run-tests: add a way to list tests, with JSON and XUnit support
authorSiddharth Agarwal <sid0@fb.com>
Tue, 06 Jun 2017 14:38:59 -0700
changeset 32704 1270b00a385d
parent 32703 9d1d3199382e
child 32705 70a020daf0b9
run-tests: add a way to list tests, with JSON and XUnit support Some test runners are interested in listing tests, so they can do their own filtering on top (usually based on attributes like historically observed runtime). Add support for that.
tests/run-tests.py
tests/test-run-tests.t
--- a/tests/run-tests.py	Tue Jun 06 13:56:53 2017 -0700
+++ b/tests/run-tests.py	Tue Jun 06 14:38:59 2017 -0700
@@ -263,6 +263,8 @@
         help="keep temporary directory after running tests")
     parser.add_option("-k", "--keywords",
         help="run tests matching keywords")
+    parser.add_option("--list-tests", action="store_true",
+        help="list tests instead of running them")
     parser.add_option("-l", "--local", action="store_true",
         help="shortcut for --with-hg=<testdir>/../hg, "
              "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set")
@@ -1907,6 +1909,25 @@
 
         self._runner = runner
 
+    def listtests(self, test):
+        result = TestResult(self._runner.options, self.stream,
+                            self.descriptions, 0)
+        test = sorted(test, key=lambda t: t.name)
+        for t in test:
+            print(t.name)
+            result.addSuccess(t)
+
+        if self._runner.options.xunit:
+            with open(self._runner.options.xunit, "wb") as xuf:
+                self._writexunit(result, xuf)
+
+        if self._runner.options.json:
+            jsonpath = os.path.join(self._runner._testdir, b'report.json')
+            with open(jsonpath, 'w') as fp:
+                self._writejson(result, fp)
+
+        return result
+
     def run(self, test):
         result = TestResult(self._runner.options, self.stream,
                             self.descriptions, self.verbosity)
@@ -2384,16 +2405,19 @@
                 verbosity = 2
             runner = TextTestRunner(self, verbosity=verbosity)
 
-            if self._installdir:
-                self._installhg()
-                self._checkhglib("Testing")
+            if self.options.list_tests:
+                result = runner.listtests(suite)
             else:
-                self._usecorrectpython()
-            if self.options.chg:
-                assert self._installdir
-                self._installchg()
+                if self._installdir:
+                    self._installhg()
+                    self._checkhglib("Testing")
+                else:
+                    self._usecorrectpython()
+                if self.options.chg:
+                    assert self._installdir
+                    self._installchg()
 
-            result = runner.run(suite)
+                result = runner.run(suite)
 
             if result.failures:
                 failed = True
--- a/tests/test-run-tests.t	Tue Jun 06 13:56:53 2017 -0700
+++ b/tests/test-run-tests.t	Tue Jun 06 14:38:59 2017 -0700
@@ -224,6 +224,60 @@
   test-failure-unicode.t * (glob)
   test-failure.t * (glob)
   test-success.t * (glob)
+
+  $ rt --list-tests
+  test-failure-unicode.t
+  test-failure.t
+  test-success.t
+
+  $ rt --list-tests --json
+  test-failure-unicode.t
+  test-failure.t
+  test-success.t
+  $ cat report.json
+  testreport ={
+      "test-failure-unicode.t": {
+          "result": "success"
+      },
+      "test-failure.t": {
+          "result": "success"
+      },
+      "test-success.t": {
+          "result": "success"
+      }
+  } (no-eol)
+
+  $ rt --list-tests --xunit=xunit.xml
+  test-failure-unicode.t
+  test-failure.t
+  test-success.t
+  $ cat xunit.xml
+  <?xml version="1.0" encoding="utf-8"?>
+  <testsuite errors="0" failures="0" name="run-tests" skipped="0" tests="0">
+    <testcase name="test-failure-unicode.t"/>
+    <testcase name="test-failure.t"/>
+    <testcase name="test-success.t"/>
+  </testsuite>
+
+  $ rt --list-tests test-failure* --json --xunit=xunit.xml
+  test-failure-unicode.t
+  test-failure.t
+  $ cat report.json
+  testreport ={
+      "test-failure-unicode.t": {
+          "result": "success"
+      },
+      "test-failure.t": {
+          "result": "success"
+      }
+  } (no-eol)
+  $ cat xunit.xml
+  <?xml version="1.0" encoding="utf-8"?>
+  <testsuite errors="0" failures="0" name="run-tests" skipped="0" tests="0">
+    <testcase name="test-failure-unicode.t"/>
+    <testcase name="test-failure.t"/>
+  </testsuite>
+
   $ rm test-failure-unicode.t
 
 test for --retest