# HG changeset patch # User Pierre-Yves David # Date 1498218574 -7200 # Node ID e9d325cfe07194b7fd0a4394e1a2157b7a05e457 # Parent 22ab466480ea4818ecf12da9fb4632301d3bd6a5 revlog: add an experimental option to mitigated delta issues (issue5480) The general delta heuristic to select a delta do not scale with the number of branch. The delta base is frequently too far away to be able to reuse a chain according to the "distance" criteria. This leads to insertion of larger delta (or even full text) that themselves push the bases for the next delta further away leading to more large deltas and full texts. This full text and frequent recomputation throw Mercurial performance in disarray. For example of a slightly large repository 280 000 files (2 150 000 versions) 430 000 changesets (10 000 topological heads) Number below compares repository with and without the distance criteria: manifest size: with: 21.4 GB without: 0.3 GB store size: with: 28.7 GB without 7.4 GB bundle last 15 00 revisions: with: 800 seconds 971 MB without: 50 seconds 73 MB unbundle time (of the last 15K revisions): with: 1150 seconds (~19 minutes) without: 35 seconds Similar issues has been observed in other repositories. Adding a new option or "feature" on stable is uncommon. However, given that this issues is making Mercurial practically unusable, I'm exceptionally targeting this patch for stable. What is actually needed is a full rework of the delta building and reading logic. However, that will be a longer process and churn not suitable for stable. In the meantime, we introduces a quick and dirty mitigation of this in the 'experimental' config space. The new option introduces a way to set the maximum amount of memory usable to store a diff in memory. This extend the ability for Mercurial to create chains without removing all safe guard regarding memory access. The option should be phased out when core has a more proper solution available. Setting the limit to '0' remove all limits, setting it to '-1' use the default limit (textsize x 4). diff -r 22ab466480ea -r e9d325cfe071 mercurial/localrepo.py --- a/mercurial/localrepo.py Tue Jun 27 18:13:10 2017 +0200 +++ b/mercurial/localrepo.py Fri Jun 23 13:49:34 2017 +0200 @@ -432,6 +432,9 @@ 'aggressivemergedeltas', False) self.svfs.options['aggressivemergedeltas'] = aggressivemergedeltas self.svfs.options['lazydeltabase'] = not scmutil.gddeltaconfig(self.ui) + chainspan = self.ui.configbytes('experimental', 'maxdeltachainspan', -1) + if 0 <= chainspan: + self.svfs.options['maxdeltachainspan'] = chainspan for r in self.requirements: if r.startswith('exp-compression-'): diff -r 22ab466480ea -r e9d325cfe071 mercurial/revlog.py --- a/mercurial/revlog.py Tue Jun 27 18:13:10 2017 +0200 +++ b/mercurial/revlog.py Fri Jun 23 13:49:34 2017 +0200 @@ -282,6 +282,7 @@ self._nodecache = {nullid: nullrev} self._nodepos = None self._compengine = 'zlib' + self._maxdeltachainspan = -1 v = REVLOG_DEFAULT_VERSION opts = getattr(opener, 'options', None) @@ -300,6 +301,8 @@ self._lazydeltabase = bool(opts.get('lazydeltabase', False)) if 'compengine' in opts: self._compengine = opts['compengine'] + if 'maxdeltachainspan' in opts: + self._maxdeltachainspan = opts['maxdeltachainspan'] if self._chunkcachesize <= 0: raise RevlogError(_('revlog chunk cache size %r is not greater ' @@ -1596,7 +1599,13 @@ # - 'compresseddeltalen' is the sum of the total size of deltas we need # to apply -- bounding it limits the amount of CPU we consume. dist, l, data, base, chainbase, chainlen, compresseddeltalen = d - if (dist > textlen * 4 or l > textlen or + + defaultmax = textlen * 4 + maxdist = self._maxdeltachainspan + if not maxdist: + maxdist = dist # ensure the conditional pass + maxdist = max(maxdist, defaultmax) + if (dist > maxdist or l > textlen or compresseddeltalen > textlen * 2 or (self._maxchainlen and chainlen > self._maxchainlen)): return False diff -r 22ab466480ea -r e9d325cfe071 tests/test-generaldelta.t --- a/tests/test-generaldelta.t Tue Jun 27 18:13:10 2017 +0200 +++ b/tests/test-generaldelta.t Fri Jun 23 13:49:34 2017 +0200 @@ -159,3 +159,191 @@ 1c5d4dc9a8b8d6e1750966d343e94db665e7a1e9 $ cd .. + +test maxdeltachainspan + + $ hg init source-repo + $ cd source-repo + $ hg debugbuilddag --new-file '.+5:brancha$.+11:branchb$.+30:branchc