changeset 32316:7340465bd788

run-tests: support multiple cases in .t test

Sometimes we want to run similar tests with slightly different
configurations. Previously we duplicate the test files. This patch
introduces special "#testcases" syntax that allows a single .t file to
contain multiple test cases.

Defined cases could be tested using "#if".

For example, if a test should behave the same with or without an
experimental flag, we can add the following to the .t header:

    #testcases default experimental-a
    #if experimental-a
      $ cat >> $HGRCPATH << EOF
      > [experimental]
      > feature=a
      > EOF
    #endif

The "experimental-a" block won't be executed when running the "default" test
case.

--- a/tests/run-tests.py	Wed May 17 19:52:18 2017 -0700
+++ b/tests/run-tests.py	Tue May 16 23:10:31 2017 -0700
@@ -216,6 +216,22 @@
         f.close()
     return entries
 
+def parsettestcases(path):
+    """read a .t test file, return a set of test case names
+
+    If path does not exist, return an empty set.
+    """
+    cases = set()
+    try:
+        with open(path, 'rb') as f:
+            for l in f:
+                if l.startswith(b'#testcases '):
+                    cases.update(l[11:].split())
+    except IOError as ex:
+        if ex.errno != errno.ENOENT:
+            raise
+    return cases
+
 def getparser():
     """Obtain the OptionParser used by the CLI."""
     parser = optparse.OptionParser("%prog [options] [tests]")
@@ -587,6 +603,7 @@
         self.bname = os.path.basename(path)
         self.name = _strpath(self.bname)
         self._testdir = os.path.dirname(path)
+        self._tmpname = os.path.basename(path)
         self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname)
 
         self._threadtmp = tmpdir
@@ -646,7 +663,7 @@
             if e.errno != errno.EEXIST:
                 raise
 
-        name = os.path.basename(self.path)
+        name = self._tmpname
         self._testtmp = os.path.join(self._threadtmp, name)
         os.mkdir(self._testtmp)
 
@@ -1055,6 +1072,19 @@
     ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256))
     ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
 
+    def __init__(self, path, *args, **kwds):
+        # accept an extra "case" parameter
+        case = None
+        if 'case' in kwds:
+            case = kwds.pop('case')
+        self._case = case
+        self._allcases = parsettestcases(path)
+        super(TTest, self).__init__(path, *args, **kwds)
+        if case:
+            self.name = '%s (case %s)' % (self.name, _strpath(case))
+            self.errpath = b'%s.%s.err' % (self.errpath[:-4], case)
+            self._tmpname += b'-%s' % case
+
     @property
     def refpath(self):
         return os.path.join(self._testdir, self.bname)
@@ -1110,6 +1140,20 @@
             self._timeout = self._slowtimeout
         return True, None
 
+    def _iftest(self, args):
+        # implements "#if"
+        reqs = []
+        for arg in args:
+            if arg.startswith(b'no-') and arg[3:] in self._allcases:
+                if arg[3:] == self._case:
+                    return False
+            elif arg in self._allcases:
+                if arg != self._case:
+                    return False
+            else:
+                reqs.append(arg)
+        return self._hghave(reqs)[0]
+
     def _parsetest(self, lines):
         # We generate a shell script which outputs unique markers to line
         # up script results with our source. These markers include input
@@ -1167,7 +1211,7 @@
                     after.setdefault(pos, []).append('  !!! invalid #if\n')
                 if skipping is not None:
                     after.setdefault(pos, []).append('  !!! nested #if\n')
-                skipping = not self._hghave(lsplit[1:])[0]
+                skipping = not self._iftest(lsplit[1:])
                 after.setdefault(pos, []).append(l)
             elif l.startswith(b'#else'):
                 if skipping is None:
@@ -2263,14 +2307,29 @@
             else:
                 args = os.listdir(b'.')
 
-        return [{'path': t} for t in args
-                if os.path.basename(t).startswith(b'test-')
-                    and (t.endswith(b'.py') or t.endswith(b'.t'))]
+        tests = []
+        for t in args:
+            if not (os.path.basename(t).startswith(b'test-')
+                    and (t.endswith(b'.py') or t.endswith(b'.t'))):
+                continue
+            if t.endswith(b'.t'):
+                # .t file may contain multiple test cases
+                cases = sorted(parsettestcases(t))
+                if cases:
+                    tests += [{'path': t, 'case': c} for c in sorted(cases)]
+                else:
+                    tests.append({'path': t})
+            else:
+                tests.append({'path': t})
+        return tests
 
     def _runtests(self, testdescs):
         def _reloadtest(test, i):
             # convert a test back to its description dict
             desc = {'path': test.path}
+            case = getattr(test, '_case', None)
+            if case:
+                desc['case'] = case
             return self._gettest(desc, i)
 
         try:
@@ -2286,7 +2345,12 @@
             if self.options.restart:
                 orig = list(testdescs)
                 while testdescs:
-                    if os.path.exists(testdescs[0]['path'] + ".err"):
+                    desc = testdescs[0]
+                    if 'case' in desc:
+                        errpath = b'%s.%s.err' % (desc['path'], desc['case'])
+                    else:
+                        errpath = b'%s.err' % desc['path']
+                    if os.path.exists(errpath):
                         break
                     testdescs.pop(0)
                 if not testdescs:
@@ -2369,6 +2433,9 @@
         refpath = os.path.join(self._testdir, path)
         tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
 
+        # extra keyword parameters. 'case' is used by .t tests
+        kwds = dict((k, testdesc[k]) for k in ['case'] if k in testdesc)
+
         t = testcls(refpath, tmpdir,
                     keeptmpdir=self.options.keep_tmpdir,
                     debug=self.options.debug,
@@ -2379,7 +2446,7 @@
                     shell=self.options.shell,
                     hgcommand=self._hgcommand,
                     usechg=bool(self.options.with_chg or self.options.chg),
-                    useipv6=useipv6)
+                    useipv6=useipv6, **kwds)
         t.should_reload = True
         return t
 

--- a/tests/test-run-tests.t	Wed May 17 19:52:18 2017 -0700
+++ b/tests/test-run-tests.t	Tue May 16 23:10:31 2017 -0700
@@ -900,3 +900,78 @@
   # Ran 1 tests, 0 skipped, 0 warned, 1 failed.
   python hash seed: * (glob)
   [1]
+
+  $ cd ..
+
+Test cases in .t files
+======================
+  $ mkdir cases
+  $ cd cases
+  $ cat > test-cases-abc.t <<'EOF'
+  > #testcases A B C
+  >   $ V=B
+  > #if A
+  >   $ V=A
+  > #endif
+  > #if C
+  >   $ V=C
+  > #endif
+  >   $ echo $V | sed 's/A/C/'
+  >   C
+  > #if C
+  >   $ [ $V = C ]
+  > #endif
+  > #if A
+  >   $ [ $V = C ]
+  >   [1]
+  > #endif
+  > #if no-C
+  >   $ [ $V = C ]
+  >   [1]
+  > #endif
+  >   $ [ $V = D ]
+  >   [1]
+  > EOF
+  $ rt
+  .
+  --- $TESTTMP/anothertests/cases/test-cases-abc.t
+  +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
+  @@ -7,7 +7,7 @@
+     $ V=C
+   #endif
+     $ echo $V | sed 's/A/C/'
+  -  C
+  +  B
+   #if C
+     $ [ $V = C ]
+   #endif
+  
+  ERROR: test-cases-abc.t (case B) output changed
+  !.
+  Failed test-cases-abc.t (case B): output changed
+  # Ran 3 tests, 0 skipped, 0 warned, 1 failed.
+  python hash seed: * (glob)
+  [1]
+
+--restart works
+
+  $ rt --restart
+  
+  --- $TESTTMP/anothertests/cases/test-cases-abc.t
+  +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
+  @@ -7,7 +7,7 @@
+     $ V=C
+   #endif
+     $ echo $V | sed 's/A/C/'
+  -  C
+  +  B
+   #if C
+     $ [ $V = C ]
+   #endif
+  
+  ERROR: test-cases-abc.t (case B) output changed
+  !.
+  Failed test-cases-abc.t (case B): output changed
+  # Ran 2 tests, 0 skipped, 0 warned, 1 failed.
+  python hash seed: * (glob)
+  [1]