changeset 30796:168ef0a4eb3b

perf: support multiple compression engines in perfrevlogchunks

Now that the revlog has a reference to a compressor, it is
possible to swap in other compression engines. So, teach
`hg perfrevlogchunks` to do that.

The default behavior of `hg perfrevlogchunks` is now to measure the
compression performance of all compression engines implementing the
revlog compressor API. This effectively adds the no-op "none"
compressor and zstd (when available) into the default set.

While we can't yet plug alternate compressors into revlogs, this
command gives us a preview of the performance. On the mozilla-unified
repository:

$ hg perfrevlogchunks -c
! compress w/ none
! wall 0.115159 comb 0.110000 user 0.110000 sys 0.000000 (best of 86)
! compress w/ zlib
! wall 5.681406 comb 5.680000 user 5.680000 sys 0.000000 (best of 3)
! compress w/ zstd
! wall 2.624781 comb 2.620000 user 2.620000 sys 0.000000 (best of 4)

$ hg perfrevlogchunks -m
! compress w/ none
! wall 0.124486 comb 0.120000 user 0.120000 sys 0.000000 (best of 79)
! compress w/ zlib
! wall 10.144701 comb 10.150000 user 10.150000 sys 0.000000 (best of 3)
! compress w/ zstd
! wall 4.383118 comb 4.390000 user 4.390000 sys 0.000000 (best of 3)

Those numbers for zstd look promising. But they aren't the full story.
For that, we'll need to look at decompression times and storage sizes.
Stay tuned...

--- a/contrib/perf.py	Mon Jan 02 11:22:52 2017 -0800
+++ b/contrib/perf.py	Mon Jan 02 12:02:08 2017 -0800
@@ -859,9 +859,10 @@
     fm.end()
 
 @command('perfrevlogchunks', revlogopts + formatteropts +
-         [('s', 'startrev', 0, 'revision to start at')],
+         [('e', 'engines', '', 'compression engines to use'),
+          ('s', 'startrev', 0, 'revision to start at')],
          '-c|-m|FILE')
-def perfrevlogchunks(ui, repo, file_=None, startrev=0, **opts):
+def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
@@ -874,6 +875,26 @@
     see ``perfrevlog`` and ``perfrevlogrevision``.
     """
     rl = cmdutil.openrevlog(repo, 'perfrevlogchunks', file_, opts)
+
+    # Verify engines argument.
+    if engines:
+        engines = set(e.strip() for e in engines.split(','))
+        for engine in engines:
+            try:
+                util.compressionengines[engine]
+            except KeyError:
+                raise error.Abort('unknown compression engine: %s' % engine)
+    else:
+        engines = []
+        for e in util.compengines:
+            engine = util.compengines[e]
+            try:
+                if engine.available():
+                    engine.revlogcompressor().compress('dummy')
+                    engines.append(e)
+            except NotImplementedError:
+                pass
+
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
@@ -916,10 +937,17 @@
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
-    def docompress():
+    def docompress(compressor):
         rl.clearcaches()
-        for chunk in chunks[0]:
-            rl.compress(chunk)
+
+        try:
+            # Swap in the requested compression engine.
+            oldcompressor = rl._compressor
+            rl._compressor = compressor
+            for chunk in chunks[0]:
+                rl.compress(chunk)
+        finally:
+            rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), 'read'),
@@ -928,9 +956,13 @@
         (lambda: doreadbatchcachedfh(), 'read batch w/ reused fd'),
         (lambda: dochunk(), 'chunk'),
         (lambda: dochunkbatch(), 'chunk batch'),
-        (lambda: docompress(), 'compress'),
     ]
 
+    for engine in sorted(engines):
+        compressor = util.compengines[engine].revlogcompressor()
+        benches.append((functools.partial(docompress, compressor),
+                        'compress w/ %s' % engine))
+
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)