revlog: use mmap by default is pre-population is available
authorPierre-Yves David <pierre-yves.david@octobus.net>
Mon, 08 Jul 2024 15:48:34 +0200
changeset 51683 5f37c36f36b9
parent 51682 c10fa6388dbf
child 51684 20e2a20674dc
revlog: use mmap by default is pre-population is available Using mmap has a great impact of memory usage on server, and a good impact on performance in multiple case. Now that we pre-populate memory mapping by default, there is case where it using mmap is slower. So we use it by default (if pre-population is available). Further work to reduce the performance impact of the pre-population will be done later. Some benchmark below (using the same setup as 522b4d729e89): As for 522b4d729e89 the impact on small repository like Mercurial or Pypy is tiny, ~1% best. However for large repositories we see some performance improvement without seeing the performance regression that we could have without pre-populate. ##### For netbeans ### data-env-vars.name = netbeans-2018-08-01-zstd-sparse-revlog ## benchmark.name = hg.command.log # bin-env-vars.hg.flavor = rust # benchmark.variants.limit-rev = 1 # benchmark.variants.patch = yes no-mmap: 0.171579 mmap: 0.166311 (-3.07%, -0.01) # bin-env-vars.hg.flavor = default no-mmap: 0.170716 mmap: 0.165218 (-3.22%, -0.01) # benchmark.variants.patch = no # benchmark.variants.rev = tip no-mmap: 0.140862 mmap: 0.137566 (-2.34%, -0.00) ## benchmark.name = hg.command.unbundle # bin-env-vars.hg.flavor = rust # benchmark.variants.issue6528 = disabled # benchmark.variants.reuse-external-delta-parent = yes # benchmark.variants.revs = any-1-extra-rev # benchmark.variants.source = unbundle no-mmap: 0.238038 mmap: 0.239912 no-populate: 0.cbd4c9 (+11.71%, +0.03) #### For Mozilla ### data-env-vars.name = mozilla-try-2019-02-18-ds2-pnm # benchmark.name = hg.command.log # bin-env-vars.hg.flavor = rust # bin-env-vars.hg.py-re2-module = default # benchmark.variants.limit-rev = 1 # benchmark.variants.patch = yes no-mmap: 0.258440 mmap: 0.237813 (-7.98%, -0.02) # benchmark.variants.limit-rev = 10 no-mmap: 1.235323 mmap: 1.213578 (-1.76%, -0.02) ## benchmark.name = hg.command.push # bin-env-vars.hg.flavor = rust # bin-env-vars.hg.py-re2-module = default # benchmark.variants.explicit-rev = none # benchmark.variants.issue6528 = disabled # benchmark.variants.protocol = ssh # benchmark.variants.reuse-external-delta-parent = yes # benchmark.variants.revs = any-1-extra-rev no-mmap: 4.790135 mmap: 4.668971 (-2.53%, -0.12) no-populate: 4.841141 (+1.06%, +0.05) ### data-env-vars.name = mozilla-try-2019-02-18-zstd-sparse-revlog ## benchmark.name = hg.command.log # bin-env-vars.hg.flavor = default # benchmark.variants.limit-rev = 1000 # benchmark.variants.rev = tip no-mmap: 0.206187 mmap: 0.197348 (-4.29%, -0.01) ## benchmark.name = hg.command.push # bin-env-vars.hg.flavor = default # benchmark.variants.explicit-rev = none # benchmark.variants.issue6528 = disabled # benchmark.variants.protocol = ssh # benchmark.variants.reuse-external-delta-parent = yes # benchmark.variants.revs = any-1-extra-rev no-mmap: 4.768259 mmap: 4.798632 no-populate: 4.953295 (+3.88%, +0.19) # benchmark.variants.revs = any-100-extra-rev no-mmap: 4.785946 mmap: 4.903618 no-populate: 5.014963 (+4.79%, +0.23) ## benchmark.name = hg.command.unbundle # bin-env-vars.hg.flavor = default # benchmark.variants.issue6528 = disabled # benchmark.variants.reuse-external-delta-parent = yes # benchmark.variants.revs = any-1-extra-rev # benchmark.variants.source = unbundle no-mmap: 1.400121 mmap: 1.423411 no-populate: 1.585365 (+13.23%, +0.19)
mercurial/configitems.toml
mercurial/localrepo.py
mercurial/util.py
--- a/mercurial/configitems.toml	Mon Jul 08 17:02:27 2024 +0200
+++ b/mercurial/configitems.toml	Mon Jul 08 15:48:34 2024 +0200
@@ -2164,7 +2164,7 @@
 [[items]]
 section = "storage"
 name = "revlog.mmap.index"
-default = false
+default-type = "dynamic"
 
 [[items]]
 section = "storage"
--- a/mercurial/localrepo.py	Mon Jul 08 17:02:27 2024 +0200
+++ b/mercurial/localrepo.py	Mon Jul 08 15:48:34 2024 +0200
@@ -1123,7 +1123,8 @@
     if 0 <= chainspan:
         delta_config.max_deltachain_span = chainspan
 
-    if ui.configbool(b'storage', b'revlog.mmap.index'):
+    has_populate = util.has_mmap_populate()
+    if ui.configbool(b'storage', b'revlog.mmap.index', has_populate):
         data_config.mmap_index_threshold = ui.configbytes(
             b'storage',
             b'revlog.mmap.index:size-threshold',
--- a/mercurial/util.py	Mon Jul 08 17:02:27 2024 +0200
+++ b/mercurial/util.py	Mon Jul 08 15:48:34 2024 +0200
@@ -440,6 +440,10 @@
         return data
 
 
+def has_mmap_populate():
+    return hasattr(mmap, 'MAP_POPULATE')
+
+
 def mmapread(fp, size=None, pre_populate=True):
     """Read a file content using mmap