Mercurial > hg
changeset 52215:36e4d1457c6b stable
run-tests: implement crude sharding support
It will help to spread the testing load across more CI runners.
| author | Pierre-Yves David <pierre-yves.david@octobus.net> |
|---|---|
| date | Thu, 22 Sep 2022 01:02:06 +0200 |
| parents | b3e696e879fc |
| children | fa58f4f97337 |
| files | tests/run-tests.py tests/test-run-tests.t |
| diffstat | 2 files changed, 160 insertions(+), 2 deletions(-) [+] |
line wrap: on
line diff
--- a/tests/run-tests.py Fri Nov 08 21:41:02 2024 +0100 +++ b/tests/run-tests.py Thu Sep 22 01:02:06 2022 +0200 @@ -447,6 +447,18 @@ help="skip tests listed in the specified blacklist file", ) selection.add_argument( + "--shard-total", + type=int, + default=None, + help="total number of shard to use (enable sharding)", + ) + selection.add_argument( + "--shard-index", + type=int, + default=None, + help="index of this shard [1-N]", + ) + selection.add_argument( "--changed", help="run tests that are changed in parent rev or working directory", ) @@ -885,6 +897,32 @@ if options.showchannels: options.nodiff = True + if options.shard_total is not None: + if options.shard_index is None: + parser.error("--shard-total requires --shard-index to be set") + + if options.shard_index is not None: + if options.shard_total is None: + parser.error("--shard-index requires --shard-total to be set") + elif options.shard_index <= 0: + msg = "--shard-index must be > 0 (%d)" + msg %= options.shard_index + parser.error(msg) + elif options.shard_index > options.shard_total: + msg = ( + "--shard-index must be <= than --shard-total (%d not in [1,%d])" + ) + msg %= (options.shard_index, options.shard_total) + parser.error(msg) + + if options.shard_total is not None and options.order_by_runtime: + msg = "cannot use --order-by-runtime when sharding" + parser.error(msg) + + if options.shard_total is not None and options.random: + msg = "cannot use --random when sharding" + parser.error(msg) + return options @@ -3157,7 +3195,11 @@ import statprof statprof.start() - result = self._run(testdescs) + result = self._run( + testdescs, + shard_index=options.shard_index, + shard_total=options.shard_total, + ) if options.profile_runner: statprof.stop() statprof.display() @@ -3166,7 +3208,7 @@ finally: os.umask(oldmask) - def _run(self, testdescs): + def _run(self, testdescs, shard_index=None, shard_total=None): testdir = getcwdb() # assume all tests in same folder for now if testdescs: @@ -3454,6 +3496,14 @@ ) vlog("# Writing to directory", _bytes2sys(self._outputdir)) + if shard_total is not None: + slot = shard_index - 1 + testdescs = [ + t + for (idx, t) in enumerate(testdescs) + if (idx % shard_total == slot) + ] + try: return self._runtests(testdescs) or 0 finally:
--- a/tests/test-run-tests.t Fri Nov 08 21:41:02 2024 +0100 +++ b/tests/test-run-tests.t Thu Sep 22 01:02:06 2022 +0200 @@ -2087,3 +2087,111 @@ 3.* (glob) $ ./test-py.py 3.* (glob) + +Test sharding +============= + + $ rt --shard-index 1 + usage: run-tests.py [options] [tests] + run-tests.py: error: --shard-index requires --shard-total to be set + [2] + $ rt --shard-total 15 + usage: run-tests.py [options] [tests] + run-tests.py: error: --shard-total requires --shard-index to be set + [2] + $ rt --shard-index -2 --shard-total 15 + usage: run-tests.py [options] [tests] + run-tests.py: error: --shard-index must be > 0 (-2) + [2] + $ rt --shard-index 10 --shard-total 5 + usage: run-tests.py [options] [tests] + run-tests.py: error: --shard-index must be <= than --shard-total (10 not in [1,5]) + [2] + $ rt --shard-index 1 --shard-total 5 + running 3 tests using 1 parallel processes + s + --- $TESTTMP/anothertests/cases/test-conditional-matching.t + +++ $TESTTMP/anothertests/cases/test-conditional-matching.t#foo.err + @@ -3,11 +3,14 @@ + richtig \(true !\) (re) + $ echo falsch + falsch \(false !\) (re) + + falsch + #if foo + $ echo arthur + arthur \(bar !\) (re) + + arthur + #endif + $ echo celeste + celeste \(foo !\) (re) + $ echo zephir + zephir \(bar !\) (re) + + zephir + + ERROR: test-conditional-matching.t#foo output changed + ! + --- $TESTTMP/anothertests/cases/test-cases-advanced-cases.t + +++ $TESTTMP/anothertests/cases/test-cases-advanced-cases.t#case-with-dashes.err + @@ -1,3 +1,3 @@ + #testcases simple case-with-dashes casewith_-.chars + $ echo $TESTCASE + - simple + + case-with-dashes + + ERROR: test-cases-advanced-cases.t#case-with-dashes output changed + ! + Skipped test-cases-abc.t: Doesn't exist + Failed test-cases-advanced-cases.t#case-with-dashes: output changed + Failed test-conditional-matching.t#foo: output changed + # Ran 2 tests, 1 skipped, 2 failed. + python hash seed: * (glob) + [1] + $ rt --shard-index 6 --shard-total 5 + usage: run-tests.py [options] [tests] + run-tests.py: error: --shard-index must be <= than --shard-total (6 not in [1,5]) + [2] + $ rt --shard-index 5 --shard-total 5 + running 3 tests using 1 parallel processes + + --- $TESTTMP/anothertests/cases/test-conditional-matching.t + +++ $TESTTMP/anothertests/cases/test-conditional-matching.t#bar.err + @@ -3,11 +3,13 @@ + richtig \(true !\) (re) + $ echo falsch + falsch \(false !\) (re) + + falsch + #if foo + $ echo arthur + arthur \(bar !\) (re) + #endif + $ echo celeste + celeste \(foo !\) (re) + + celeste + $ echo zephir + zephir \(bar !\) (re) + + ERROR: test-conditional-matching.t#bar output changed + ! + --- $TESTTMP/anothertests/cases/test-substitution.t + +++ $TESTTMP/anothertests/cases/test-substitution.t.err + @@ -7,3 +7,4 @@ + $ echo lastbar + last$YYY$ + $ echo foo-bar foo-baz + + $XXX=bar foo-baz$ + + ERROR: test-substitution.t output changed + ! + --- $TESTTMP/anothertests/cases/test-py.py.out + +++ $TESTTMP/anothertests/cases/test-py.py.err + @@ -0,0 +1* @@ (glob) + +3.* (glob) + + ERROR: test-py.py output changed + ! + Failed test-conditional-matching.t#bar: output changed + Failed test-py.py: output changed + Failed test-substitution.t: output changed + # Ran 3 tests, 0 skipped, 3 failed. + python hash seed: * (glob) + [1]