Mercurial > evolve
changeset 814:e629a4f9d498
fastobs: move the extension to the hgext dir
| author | Pierre-Yves David <pierre-yves.david@fb.com> |
|---|---|
| date | Thu, 27 Feb 2014 17:11:09 -0800 |
| parents | b49a9276ec8e |
| children | 916bebf91c41 |
| files | hgext/hgfastobs.py hgfastobs.py tests/test-boxpush.t |
| diffstat | 3 files changed, 125 insertions(+), 125 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hgext/hgfastobs.py Thu Feb 27 17:11:09 2014 -0800 @@ -0,0 +1,124 @@ +"""Extension to try and speed up transfer of obsolete markers. + +Mercurial 2.6 transfers obsolete markers in the dumbest way possible: +it simply transfers all of them to the server on every +operation. While this /works/, it's not ideal because it's a large +amount of extra data for users to pull down (1.9M for the 17k obsolete +markers in hg-crew as of this writing in late July 2013). It's also +frustrating because this transfer takes a nontrivial amount of time. + +You can specify a strategy with the config knob +obsolete.syncstrategy. Current strategies are "stock" and +"boxfill". Default strategy is presently boxfill. + +:stock: use the default strategy of mercurial explaned above + +:boxfill: transmit obsolete markers which list any of transmitted changesets as + a successor (transitively), as well as any kill markers for dead + nodes descended from any of the precursors of outgoing.missing. + +TODO(durin42): consider better names for sync strategies. +""" +import sys + +from mercurial import base85 +from mercurial import commands +from mercurial import extensions +from mercurial import node +from mercurial import obsolete +from mercurial import exchange +from mercurial import revset +from mercurial.i18n import _ + +_strategies = { + 'stock': exchange._pushobsolete, + } + +def _strategy(name, default=False): + def inner(func): + _strategies[name] = func + if default: + _strategies[None] = func + return func + return inner + +def _pushobsoletewrapper(orig, pushop): + stratfn = _strategies[pushop.repo.ui.config('obsolete', 'syncstrategy')] + return stratfn(pushop) + +extensions.wrapfunction(exchange, '_pushobsolete', _pushobsoletewrapper) + +def _precursors(repo, s): + """Precursor of a changeset""" + cs = set() + nm = repo.changelog.nodemap + markerbysubj = repo.obsstore.precursors + for r in s: + for p in markerbysubj.get(repo[r].node(), ()): + pr = nm.get(p[0]) + if pr is not None: + cs.add(pr) + return cs + +def _revsetprecursors(repo, subset, x): + s = revset.getset(repo, revset.baseset(range(len(repo))), x) + cs = _precursors(repo, s) + return revset.baseset([r for r in subset if r in cs]) + +revset.symbols['_fastobs_precursors'] = _revsetprecursors + + +@_strategy('boxfill', default=True) +def boxfill(pushop): + """The "fill in the box" strategy from the 2.6 sprint. + + See the notes[0] from the 2.6 sprint for what "fill in the box" + means here. It's a fairly subtle algorithm, which may have + surprising behavior at times, but was the least-bad option + proposed at the sprint. + + [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt + """ + repo = pushop.repo + remote = pushop.remote + outgoing = pushop.outgoing + urepo = pushop.repo.unfiltered() + # need to collect obsolete markers which list any of + # outgoing.missing as a successor (transitively), as well as any + # kill markers for dead nodes descended from any of the precursors + # of outgoing.missing. + boxedges = urepo.revs( + '(descendants(_fastobs_precursors(%ln)) or ' + ' descendants(%ln)) and hidden()', + outgoing.missing, outgoing.missing) + transmit = [] + for node in outgoing.missing: + transmit.extend(obsolete.precursormarkers(urepo[node])) + for node in boxedges: + transmit.extend(obsolete.successormarkers(urepo[node])) + transmit = list(set(transmit)) + xmit, total = len(transmit), len(repo.obsstore._all) + repo.ui.status( + 'boxpush: about to transmit %d obsolete markers (%d markers total)\n' + % (xmit, total)) + parts, size, chunk = [], 0, 0 + def transmitmarks(): + repo.ui.note( + 'boxpush: sending a chunk of obsolete markers\n') + data = ''.join([obsolete._pack('>B', obsolete._fmversion)] + parts) + remote.pushkey('obsolete', 'dump-%d' % chunk, '', + base85.b85encode(data)) + + for marker in transmit: + enc = obsolete._encodeonemarker(_markertuple(marker)) + parts.append(enc) + size += len(enc) + if size > obsolete._maxpayload: + transmitmarks() + parts, size = [], 0 + chunk += 1 + if parts: + transmitmarks() + +def _markertuple(marker): + return marker._data
--- a/hgfastobs.py Wed Feb 26 15:25:33 2014 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,124 +0,0 @@ -"""Extension to try and speed up transfer of obsolete markers. - -Mercurial 2.6 transfers obsolete markers in the dumbest way possible: -it simply transfers all of them to the server on every -operation. While this /works/, it's not ideal because it's a large -amount of extra data for users to pull down (1.9M for the 17k obsolete -markers in hg-crew as of this writing in late July 2013). It's also -frustrating because this transfer takes a nontrivial amount of time. - -You can specify a strategy with the config knob -obsolete.syncstrategy. Current strategies are "stock" and -"boxfill". Default strategy is presently boxfill. - -:stock: use the default strategy of mercurial explaned above - -:boxfill: transmit obsolete markers which list any of transmitted changesets as - a successor (transitively), as well as any kill markers for dead - nodes descended from any of the precursors of outgoing.missing. - -TODO(durin42): consider better names for sync strategies. -""" -import sys - -from mercurial import base85 -from mercurial import commands -from mercurial import extensions -from mercurial import node -from mercurial import obsolete -from mercurial import exchange -from mercurial import revset -from mercurial.i18n import _ - -_strategies = { - 'stock': exchange._pushobsolete, - } - -def _strategy(name, default=False): - def inner(func): - _strategies[name] = func - if default: - _strategies[None] = func - return func - return inner - -def _pushobsoletewrapper(orig, pushop): - stratfn = _strategies[pushop.repo.ui.config('obsolete', 'syncstrategy')] - return stratfn(pushop) - -extensions.wrapfunction(exchange, '_pushobsolete', _pushobsoletewrapper) - -def _precursors(repo, s): - """Precursor of a changeset""" - cs = set() - nm = repo.changelog.nodemap - markerbysubj = repo.obsstore.precursors - for r in s: - for p in markerbysubj.get(repo[r].node(), ()): - pr = nm.get(p[0]) - if pr is not None: - cs.add(pr) - return cs - -def _revsetprecursors(repo, subset, x): - s = revset.getset(repo, revset.baseset(range(len(repo))), x) - cs = _precursors(repo, s) - return revset.baseset([r for r in subset if r in cs]) - -revset.symbols['_fastobs_precursors'] = _revsetprecursors - - -@_strategy('boxfill', default=True) -def boxfill(pushop): - """The "fill in the box" strategy from the 2.6 sprint. - - See the notes[0] from the 2.6 sprint for what "fill in the box" - means here. It's a fairly subtle algorithm, which may have - surprising behavior at times, but was the least-bad option - proposed at the sprint. - - [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt - """ - repo = pushop.repo - remote = pushop.remote - outgoing = pushop.outgoing - urepo = pushop.repo.unfiltered() - # need to collect obsolete markers which list any of - # outgoing.missing as a successor (transitively), as well as any - # kill markers for dead nodes descended from any of the precursors - # of outgoing.missing. - boxedges = urepo.revs( - '(descendants(_fastobs_precursors(%ln)) or ' - ' descendants(%ln)) and hidden()', - outgoing.missing, outgoing.missing) - transmit = [] - for node in outgoing.missing: - transmit.extend(obsolete.precursormarkers(urepo[node])) - for node in boxedges: - transmit.extend(obsolete.successormarkers(urepo[node])) - transmit = list(set(transmit)) - xmit, total = len(transmit), len(repo.obsstore._all) - repo.ui.status( - 'boxpush: about to transmit %d obsolete markers (%d markers total)\n' - % (xmit, total)) - parts, size, chunk = [], 0, 0 - def transmitmarks(): - repo.ui.note( - 'boxpush: sending a chunk of obsolete markers\n') - data = ''.join([obsolete._pack('>B', obsolete._fmversion)] + parts) - remote.pushkey('obsolete', 'dump-%d' % chunk, '', - base85.b85encode(data)) - - for marker in transmit: - enc = obsolete._encodeonemarker(_markertuple(marker)) - parts.append(enc) - size += len(enc) - if size > obsolete._maxpayload: - transmitmarks() - parts, size = [], 0 - chunk += 1 - if parts: - transmitmarks() - -def _markertuple(marker): - return marker._data
--- a/tests/test-boxpush.t Wed Feb 26 15:25:33 2014 -0800 +++ b/tests/test-boxpush.t Thu Feb 27 17:11:09 2014 -0800 @@ -1,4 +1,4 @@ - $ fastobs="$TESTDIR"/../hgfastobs.py + $ fastobs="$TESTDIR"/../hgext/hgfastobs.py $ echo 'from mercurial import obsolete ; obsolete._enabled = True' > enableobs.py $ cat >> $HGRCPATH <<EOF > [obsolete]