Mercurial > hg
comparison tests/run-tests.py @ 32316:7340465bd788
run-tests: support multiple cases in .t test
Sometimes we want to run similar tests with slightly different
configurations. Previously we duplicate the test files. This patch
introduces special "#testcases" syntax that allows a single .t file to
contain multiple test cases.
Defined cases could be tested using "#if".
For example, if a test should behave the same with or without an
experimental flag, we can add the following to the .t header:
#testcases default experimental-a
#if experimental-a
$ cat >> $HGRCPATH << EOF
> [experimental]
> feature=a
> EOF
#endif
The "experimental-a" block won't be executed when running the "default" test
case.
author | Jun Wu <quark@fb.com> |
---|---|
date | Tue, 16 May 2017 23:10:31 -0700 |
parents | a77a75a428f7 |
children | 531e6a57abd2 |
comparison
equal
deleted
inserted
replaced
32315:67026d65a4fc | 32316:7340465bd788 |
---|---|
213 if line: | 213 if line: |
214 entries[line] = filename | 214 entries[line] = filename |
215 | 215 |
216 f.close() | 216 f.close() |
217 return entries | 217 return entries |
218 | |
219 def parsettestcases(path): | |
220 """read a .t test file, return a set of test case names | |
221 | |
222 If path does not exist, return an empty set. | |
223 """ | |
224 cases = set() | |
225 try: | |
226 with open(path, 'rb') as f: | |
227 for l in f: | |
228 if l.startswith(b'#testcases '): | |
229 cases.update(l[11:].split()) | |
230 except IOError as ex: | |
231 if ex.errno != errno.ENOENT: | |
232 raise | |
233 return cases | |
218 | 234 |
219 def getparser(): | 235 def getparser(): |
220 """Obtain the OptionParser used by the CLI.""" | 236 """Obtain the OptionParser used by the CLI.""" |
221 parser = optparse.OptionParser("%prog [options] [tests]") | 237 parser = optparse.OptionParser("%prog [options] [tests]") |
222 | 238 |
585 """ | 601 """ |
586 self.path = path | 602 self.path = path |
587 self.bname = os.path.basename(path) | 603 self.bname = os.path.basename(path) |
588 self.name = _strpath(self.bname) | 604 self.name = _strpath(self.bname) |
589 self._testdir = os.path.dirname(path) | 605 self._testdir = os.path.dirname(path) |
606 self._tmpname = os.path.basename(path) | |
590 self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname) | 607 self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname) |
591 | 608 |
592 self._threadtmp = tmpdir | 609 self._threadtmp = tmpdir |
593 self._keeptmpdir = keeptmpdir | 610 self._keeptmpdir = keeptmpdir |
594 self._debug = debug | 611 self._debug = debug |
644 os.mkdir(self._threadtmp) | 661 os.mkdir(self._threadtmp) |
645 except OSError as e: | 662 except OSError as e: |
646 if e.errno != errno.EEXIST: | 663 if e.errno != errno.EEXIST: |
647 raise | 664 raise |
648 | 665 |
649 name = os.path.basename(self.path) | 666 name = self._tmpname |
650 self._testtmp = os.path.join(self._threadtmp, name) | 667 self._testtmp = os.path.join(self._threadtmp, name) |
651 os.mkdir(self._testtmp) | 668 os.mkdir(self._testtmp) |
652 | 669 |
653 # Remove any previous output files. | 670 # Remove any previous output files. |
654 if os.path.exists(self.errpath): | 671 if os.path.exists(self.errpath): |
1053 | 1070 |
1054 ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub | 1071 ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub |
1055 ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256)) | 1072 ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256)) |
1056 ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'}) | 1073 ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'}) |
1057 | 1074 |
1075 def __init__(self, path, *args, **kwds): | |
1076 # accept an extra "case" parameter | |
1077 case = None | |
1078 if 'case' in kwds: | |
1079 case = kwds.pop('case') | |
1080 self._case = case | |
1081 self._allcases = parsettestcases(path) | |
1082 super(TTest, self).__init__(path, *args, **kwds) | |
1083 if case: | |
1084 self.name = '%s (case %s)' % (self.name, _strpath(case)) | |
1085 self.errpath = b'%s.%s.err' % (self.errpath[:-4], case) | |
1086 self._tmpname += b'-%s' % case | |
1087 | |
1058 @property | 1088 @property |
1059 def refpath(self): | 1089 def refpath(self): |
1060 return os.path.join(self._testdir, self.bname) | 1090 return os.path.join(self._testdir, self.bname) |
1061 | 1091 |
1062 def _run(self, env): | 1092 def _run(self, env): |
1107 return False, stdout | 1137 return False, stdout |
1108 | 1138 |
1109 if 'slow' in reqs: | 1139 if 'slow' in reqs: |
1110 self._timeout = self._slowtimeout | 1140 self._timeout = self._slowtimeout |
1111 return True, None | 1141 return True, None |
1142 | |
1143 def _iftest(self, args): | |
1144 # implements "#if" | |
1145 reqs = [] | |
1146 for arg in args: | |
1147 if arg.startswith(b'no-') and arg[3:] in self._allcases: | |
1148 if arg[3:] == self._case: | |
1149 return False | |
1150 elif arg in self._allcases: | |
1151 if arg != self._case: | |
1152 return False | |
1153 else: | |
1154 reqs.append(arg) | |
1155 return self._hghave(reqs)[0] | |
1112 | 1156 |
1113 def _parsetest(self, lines): | 1157 def _parsetest(self, lines): |
1114 # We generate a shell script which outputs unique markers to line | 1158 # We generate a shell script which outputs unique markers to line |
1115 # up script results with our source. These markers include input | 1159 # up script results with our source. These markers include input |
1116 # line number and the last return code. | 1160 # line number and the last return code. |
1165 lsplit = l.split() | 1209 lsplit = l.split() |
1166 if len(lsplit) < 2 or lsplit[0] != b'#if': | 1210 if len(lsplit) < 2 or lsplit[0] != b'#if': |
1167 after.setdefault(pos, []).append(' !!! invalid #if\n') | 1211 after.setdefault(pos, []).append(' !!! invalid #if\n') |
1168 if skipping is not None: | 1212 if skipping is not None: |
1169 after.setdefault(pos, []).append(' !!! nested #if\n') | 1213 after.setdefault(pos, []).append(' !!! nested #if\n') |
1170 skipping = not self._hghave(lsplit[1:])[0] | 1214 skipping = not self._iftest(lsplit[1:]) |
1171 after.setdefault(pos, []).append(l) | 1215 after.setdefault(pos, []).append(l) |
1172 elif l.startswith(b'#else'): | 1216 elif l.startswith(b'#else'): |
1173 if skipping is None: | 1217 if skipping is None: |
1174 after.setdefault(pos, []).append(' !!! missing #if\n') | 1218 after.setdefault(pos, []).append(' !!! missing #if\n') |
1175 skipping = not skipping | 1219 skipping = not skipping |
2261 stdout, stderr = proc.communicate() | 2305 stdout, stderr = proc.communicate() |
2262 args = stdout.strip(b'\0').split(b'\0') | 2306 args = stdout.strip(b'\0').split(b'\0') |
2263 else: | 2307 else: |
2264 args = os.listdir(b'.') | 2308 args = os.listdir(b'.') |
2265 | 2309 |
2266 return [{'path': t} for t in args | 2310 tests = [] |
2267 if os.path.basename(t).startswith(b'test-') | 2311 for t in args: |
2268 and (t.endswith(b'.py') or t.endswith(b'.t'))] | 2312 if not (os.path.basename(t).startswith(b'test-') |
2313 and (t.endswith(b'.py') or t.endswith(b'.t'))): | |
2314 continue | |
2315 if t.endswith(b'.t'): | |
2316 # .t file may contain multiple test cases | |
2317 cases = sorted(parsettestcases(t)) | |
2318 if cases: | |
2319 tests += [{'path': t, 'case': c} for c in sorted(cases)] | |
2320 else: | |
2321 tests.append({'path': t}) | |
2322 else: | |
2323 tests.append({'path': t}) | |
2324 return tests | |
2269 | 2325 |
2270 def _runtests(self, testdescs): | 2326 def _runtests(self, testdescs): |
2271 def _reloadtest(test, i): | 2327 def _reloadtest(test, i): |
2272 # convert a test back to its description dict | 2328 # convert a test back to its description dict |
2273 desc = {'path': test.path} | 2329 desc = {'path': test.path} |
2330 case = getattr(test, '_case', None) | |
2331 if case: | |
2332 desc['case'] = case | |
2274 return self._gettest(desc, i) | 2333 return self._gettest(desc, i) |
2275 | 2334 |
2276 try: | 2335 try: |
2277 if self._installdir: | 2336 if self._installdir: |
2278 self._installhg() | 2337 self._installhg() |
2284 self._installchg() | 2343 self._installchg() |
2285 | 2344 |
2286 if self.options.restart: | 2345 if self.options.restart: |
2287 orig = list(testdescs) | 2346 orig = list(testdescs) |
2288 while testdescs: | 2347 while testdescs: |
2289 if os.path.exists(testdescs[0]['path'] + ".err"): | 2348 desc = testdescs[0] |
2349 if 'case' in desc: | |
2350 errpath = b'%s.%s.err' % (desc['path'], desc['case']) | |
2351 else: | |
2352 errpath = b'%s.err' % desc['path'] | |
2353 if os.path.exists(errpath): | |
2290 break | 2354 break |
2291 testdescs.pop(0) | 2355 testdescs.pop(0) |
2292 if not testdescs: | 2356 if not testdescs: |
2293 print("running all tests") | 2357 print("running all tests") |
2294 testdescs = orig | 2358 testdescs = orig |
2367 break | 2431 break |
2368 | 2432 |
2369 refpath = os.path.join(self._testdir, path) | 2433 refpath = os.path.join(self._testdir, path) |
2370 tmpdir = os.path.join(self._hgtmp, b'child%d' % count) | 2434 tmpdir = os.path.join(self._hgtmp, b'child%d' % count) |
2371 | 2435 |
2436 # extra keyword parameters. 'case' is used by .t tests | |
2437 kwds = dict((k, testdesc[k]) for k in ['case'] if k in testdesc) | |
2438 | |
2372 t = testcls(refpath, tmpdir, | 2439 t = testcls(refpath, tmpdir, |
2373 keeptmpdir=self.options.keep_tmpdir, | 2440 keeptmpdir=self.options.keep_tmpdir, |
2374 debug=self.options.debug, | 2441 debug=self.options.debug, |
2375 timeout=self.options.timeout, | 2442 timeout=self.options.timeout, |
2376 startport=self._getport(count), | 2443 startport=self._getport(count), |
2377 extraconfigopts=self.options.extra_config_opt, | 2444 extraconfigopts=self.options.extra_config_opt, |
2378 py3kwarnings=self.options.py3k_warnings, | 2445 py3kwarnings=self.options.py3k_warnings, |
2379 shell=self.options.shell, | 2446 shell=self.options.shell, |
2380 hgcommand=self._hgcommand, | 2447 hgcommand=self._hgcommand, |
2381 usechg=bool(self.options.with_chg or self.options.chg), | 2448 usechg=bool(self.options.with_chg or self.options.chg), |
2382 useipv6=useipv6) | 2449 useipv6=useipv6, **kwds) |
2383 t.should_reload = True | 2450 t.should_reload = True |
2384 return t | 2451 return t |
2385 | 2452 |
2386 def _cleanup(self): | 2453 def _cleanup(self): |
2387 """Clean up state from this test invocation.""" | 2454 """Clean up state from this test invocation.""" |