run-tests: implement crude sharding support
It will help to spread the testing load across more CI runners.
--- a/tests/run-tests.py Fri Nov 08 21:41:02 2024 +0100
+++ b/tests/run-tests.py Thu Sep 22 01:02:06 2022 +0200
@@ -447,6 +447,18 @@
help="skip tests listed in the specified blacklist file",
)
selection.add_argument(
+ "--shard-total",
+ type=int,
+ default=None,
+ help="total number of shard to use (enable sharding)",
+ )
+ selection.add_argument(
+ "--shard-index",
+ type=int,
+ default=None,
+ help="index of this shard [1-N]",
+ )
+ selection.add_argument(
"--changed",
help="run tests that are changed in parent rev or working directory",
)
@@ -885,6 +897,32 @@
if options.showchannels:
options.nodiff = True
+ if options.shard_total is not None:
+ if options.shard_index is None:
+ parser.error("--shard-total requires --shard-index to be set")
+
+ if options.shard_index is not None:
+ if options.shard_total is None:
+ parser.error("--shard-index requires --shard-total to be set")
+ elif options.shard_index <= 0:
+ msg = "--shard-index must be > 0 (%d)"
+ msg %= options.shard_index
+ parser.error(msg)
+ elif options.shard_index > options.shard_total:
+ msg = (
+ "--shard-index must be <= than --shard-total (%d not in [1,%d])"
+ )
+ msg %= (options.shard_index, options.shard_total)
+ parser.error(msg)
+
+ if options.shard_total is not None and options.order_by_runtime:
+ msg = "cannot use --order-by-runtime when sharding"
+ parser.error(msg)
+
+ if options.shard_total is not None and options.random:
+ msg = "cannot use --random when sharding"
+ parser.error(msg)
+
return options
@@ -3157,7 +3195,11 @@
import statprof
statprof.start()
- result = self._run(testdescs)
+ result = self._run(
+ testdescs,
+ shard_index=options.shard_index,
+ shard_total=options.shard_total,
+ )
if options.profile_runner:
statprof.stop()
statprof.display()
@@ -3166,7 +3208,7 @@
finally:
os.umask(oldmask)
- def _run(self, testdescs):
+ def _run(self, testdescs, shard_index=None, shard_total=None):
testdir = getcwdb()
# assume all tests in same folder for now
if testdescs:
@@ -3454,6 +3496,14 @@
)
vlog("# Writing to directory", _bytes2sys(self._outputdir))
+ if shard_total is not None:
+ slot = shard_index - 1
+ testdescs = [
+ t
+ for (idx, t) in enumerate(testdescs)
+ if (idx % shard_total == slot)
+ ]
+
try:
return self._runtests(testdescs) or 0
finally:
--- a/tests/test-run-tests.t Fri Nov 08 21:41:02 2024 +0100
+++ b/tests/test-run-tests.t Thu Sep 22 01:02:06 2022 +0200
@@ -2087,3 +2087,111 @@
3.* (glob)
$ ./test-py.py
3.* (glob)
+
+Test sharding
+=============
+
+ $ rt --shard-index 1
+ usage: run-tests.py [options] [tests]
+ run-tests.py: error: --shard-index requires --shard-total to be set
+ [2]
+ $ rt --shard-total 15
+ usage: run-tests.py [options] [tests]
+ run-tests.py: error: --shard-total requires --shard-index to be set
+ [2]
+ $ rt --shard-index -2 --shard-total 15
+ usage: run-tests.py [options] [tests]
+ run-tests.py: error: --shard-index must be > 0 (-2)
+ [2]
+ $ rt --shard-index 10 --shard-total 5
+ usage: run-tests.py [options] [tests]
+ run-tests.py: error: --shard-index must be <= than --shard-total (10 not in [1,5])
+ [2]
+ $ rt --shard-index 1 --shard-total 5
+ running 3 tests using 1 parallel processes
+ s
+ --- $TESTTMP/anothertests/cases/test-conditional-matching.t
+ +++ $TESTTMP/anothertests/cases/test-conditional-matching.t#foo.err
+ @@ -3,11 +3,14 @@
+ richtig \(true !\) (re)
+ $ echo falsch
+ falsch \(false !\) (re)
+ + falsch
+ #if foo
+ $ echo arthur
+ arthur \(bar !\) (re)
+ + arthur
+ #endif
+ $ echo celeste
+ celeste \(foo !\) (re)
+ $ echo zephir
+ zephir \(bar !\) (re)
+ + zephir
+
+ ERROR: test-conditional-matching.t#foo output changed
+ !
+ --- $TESTTMP/anothertests/cases/test-cases-advanced-cases.t
+ +++ $TESTTMP/anothertests/cases/test-cases-advanced-cases.t#case-with-dashes.err
+ @@ -1,3 +1,3 @@
+ #testcases simple case-with-dashes casewith_-.chars
+ $ echo $TESTCASE
+ - simple
+ + case-with-dashes
+
+ ERROR: test-cases-advanced-cases.t#case-with-dashes output changed
+ !
+ Skipped test-cases-abc.t: Doesn't exist
+ Failed test-cases-advanced-cases.t#case-with-dashes: output changed
+ Failed test-conditional-matching.t#foo: output changed
+ # Ran 2 tests, 1 skipped, 2 failed.
+ python hash seed: * (glob)
+ [1]
+ $ rt --shard-index 6 --shard-total 5
+ usage: run-tests.py [options] [tests]
+ run-tests.py: error: --shard-index must be <= than --shard-total (6 not in [1,5])
+ [2]
+ $ rt --shard-index 5 --shard-total 5
+ running 3 tests using 1 parallel processes
+
+ --- $TESTTMP/anothertests/cases/test-conditional-matching.t
+ +++ $TESTTMP/anothertests/cases/test-conditional-matching.t#bar.err
+ @@ -3,11 +3,13 @@
+ richtig \(true !\) (re)
+ $ echo falsch
+ falsch \(false !\) (re)
+ + falsch
+ #if foo
+ $ echo arthur
+ arthur \(bar !\) (re)
+ #endif
+ $ echo celeste
+ celeste \(foo !\) (re)
+ + celeste
+ $ echo zephir
+ zephir \(bar !\) (re)
+
+ ERROR: test-conditional-matching.t#bar output changed
+ !
+ --- $TESTTMP/anothertests/cases/test-substitution.t
+ +++ $TESTTMP/anothertests/cases/test-substitution.t.err
+ @@ -7,3 +7,4 @@
+ $ echo lastbar
+ last$YYY$
+ $ echo foo-bar foo-baz
+ + $XXX=bar foo-baz$
+
+ ERROR: test-substitution.t output changed
+ !
+ --- $TESTTMP/anothertests/cases/test-py.py.out
+ +++ $TESTTMP/anothertests/cases/test-py.py.err
+ @@ -0,0 +1* @@ (glob)
+ +3.* (glob)
+
+ ERROR: test-py.py output changed
+ !
+ Failed test-conditional-matching.t#bar: output changed
+ Failed test-py.py: output changed
+ Failed test-substitution.t: output changed
+ # Ran 3 tests, 0 skipped, 3 failed.
+ python hash seed: * (glob)
+ [1]