large-files: use `running_status` in `updatestandinsbymatch`
This is the way.
# shallowbundle.py - bundle10 implementation for use with shallow repositories
#
# Copyright 2013 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from mercurial.i18n import _
from mercurial.node import bin, hex
from mercurial import (
bundlerepo,
changegroup,
error,
match,
mdiff,
)
from . import (
constants,
remotefilelog,
shallowutil,
)
NoFiles = 0
LocalFiles = 1
AllFiles = 2
def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None):
if not isinstance(rlog, remotefilelog.remotefilelog):
for c in super(cls, self).group(nodelist, rlog, lookup, units=units):
yield c
return
if len(nodelist) == 0:
yield self.close()
return
nodelist = shallowutil.sortnodes(nodelist, rlog.parents)
# add the parent of the first rev
p = rlog.parents(nodelist[0])[0]
nodelist.insert(0, p)
# build deltas
for i in range(len(nodelist) - 1):
prev, curr = nodelist[i], nodelist[i + 1]
linknode = lookup(curr)
for c in self.nodechunk(rlog, curr, prev, linknode):
yield c
yield self.close()
class shallowcg1packer(changegroup.cgpacker):
def generate(self, commonrevs, clnodes, fastpathlinkrev, source, **kwargs):
if shallowutil.isenabled(self._repo):
fastpathlinkrev = False
return super(shallowcg1packer, self).generate(
commonrevs, clnodes, fastpathlinkrev, source, **kwargs
)
def group(self, nodelist, rlog, lookup, units=None, reorder=None):
return shallowgroup(
shallowcg1packer, self, nodelist, rlog, lookup, units=units
)
def generatefiles(self, changedfiles, *args, **kwargs):
try:
linknodes, commonrevs, source = args
except ValueError:
commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
if shallowutil.isenabled(self._repo):
repo = self._repo
if isinstance(repo, bundlerepo.bundlerepository):
# If the bundle contains filelogs, we can't pull from it, since
# bundlerepo is heavily tied to revlogs. Instead require that
# the user use unbundle instead.
# Force load the filelog data.
bundlerepo.bundlerepository.file(repo, b'foo')
if repo._cgfilespos:
raise error.Abort(
b"cannot pull from full bundles",
hint=b"use `hg unbundle` instead",
)
return []
filestosend = self.shouldaddfilegroups(source)
if filestosend == NoFiles:
changedfiles = list(
[f for f in changedfiles if not repo.shallowmatch(f)]
)
return super(shallowcg1packer, self).generatefiles(
changedfiles, *args, **kwargs
)
def shouldaddfilegroups(self, source):
repo = self._repo
if not shallowutil.isenabled(repo):
return AllFiles
if source == b"push" or source == b"bundle":
return AllFiles
# We won't actually strip the files, but we should put them in any
# backup bundle generated by strip (especially for cases like narrow's
# `hg tracked --removeinclude`, as failing to do so means that the
# "saved" changesets during a strip won't have their files reapplied and
# thus their linknode adjusted, if necessary).
if source == b"strip":
cfg = repo.ui.config(b'remotefilelog', b'strip.includefiles')
if cfg == b'local':
return LocalFiles
elif cfg != b'none':
return AllFiles
caps = self._bundlecaps or []
if source == b"serve" or source == b"pull":
if constants.BUNDLE2_CAPABLITY in caps:
return LocalFiles
else:
# Serving to a full repo requires us to serve everything
repo.ui.warn(_(b"pulling from a shallow repo\n"))
return AllFiles
return NoFiles
def prune(self, rlog, missing, commonrevs):
if not isinstance(rlog, remotefilelog.remotefilelog):
return super(shallowcg1packer, self).prune(
rlog, missing, commonrevs
)
repo = self._repo
results = []
for fnode in missing:
fctx = repo.filectx(rlog.filename, fileid=fnode)
if fctx.linkrev() not in commonrevs:
results.append(fnode)
return results
def nodechunk(self, revlog, node, prevnode, linknode):
prefix = b''
if prevnode == revlog.nullid:
delta = revlog.rawdata(node)
prefix = mdiff.trivialdiffheader(len(delta))
else:
# Actually uses remotefilelog.revdiff which works on nodes, not revs
delta = revlog.revdiff(prevnode, node)
p1, p2 = revlog.parents(node)
flags = revlog.flags(node)
meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags)
meta += prefix
l = len(meta) + len(delta)
yield changegroup.chunkheader(l)
yield meta
yield delta
def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
if not shallowutil.isenabled(repo):
return orig(repo, outgoing, version, source, *args, **kwargs)
original = repo.shallowmatch
try:
# if serving, only send files the clients has patterns for
if source == b'serve':
bundlecaps = kwargs.get('bundlecaps')
includepattern = None
excludepattern = None
for cap in bundlecaps or []:
if cap.startswith(b"includepattern="):
raw = cap[len(b"includepattern=") :]
if raw:
includepattern = raw.split(b'\0')
elif cap.startswith(b"excludepattern="):
raw = cap[len(b"excludepattern=") :]
if raw:
excludepattern = raw.split(b'\0')
if includepattern or excludepattern:
repo.shallowmatch = match.match(
repo.root, b'', None, includepattern, excludepattern
)
else:
repo.shallowmatch = match.always()
return orig(repo, outgoing, version, source, *args, **kwargs)
finally:
repo.shallowmatch = original
def addchangegroupfiles(
orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
):
if not shallowutil.isenabled(repo):
return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
newfiles = 0
visited = set()
revisiondatas = {}
queue = []
# Normal Mercurial processes each file one at a time, adding all
# the new revisions for that file at once. In remotefilelog a file
# revision may depend on a different file's revision (in the case
# of a rename/copy), so we must lay all revisions down across all
# files in topological order.
# read all the file chunks but don't add them
progress = repo.ui.makeprogress(_(b'files'), total=expectedfiles)
while True:
chunkdata = source.filelogheader()
if not chunkdata:
break
f = chunkdata[b"filename"]
repo.ui.debug(b"adding %s revisions\n" % f)
progress.increment()
if not repo.shallowmatch(f):
fl = repo.file(f)
deltas = source.deltaiter()
fl.addgroup(deltas, revmap, trp)
continue
chain = None
while True:
# returns: None or (
# node,
# p1,
# p2,
# cs,
# deltabase,
# delta,
# flags,
# sidedata,
# proto_flags
# )
revisiondata = source.deltachunk(chain)
if not revisiondata:
break
chain = revisiondata[0]
revisiondatas[(f, chain)] = revisiondata
queue.append((f, chain))
if f not in visited:
newfiles += 1
visited.add(f)
if chain is None:
raise error.Abort(_(b"received file revlog group is empty"))
processed = set()
def available(f, node, depf, depnode):
if depnode != repo.nullid and (depf, depnode) not in processed:
if not (depf, depnode) in revisiondatas:
# It's not in the changegroup, assume it's already
# in the repo
return True
# re-add self to queue
queue.insert(0, (f, node))
# add dependency in front
queue.insert(0, (depf, depnode))
return False
return True
skipcount = 0
# Prefetch the non-bundled revisions that we will need
prefetchfiles = []
for f, node in queue:
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags, sdata, pfl)
dependents = [revisiondata[1], revisiondata[2], revisiondata[4]]
for dependent in dependents:
if dependent == repo.nullid or (f, dependent) in revisiondatas:
continue
prefetchfiles.append((f, hex(dependent)))
repo.fileservice.prefetch(prefetchfiles)
# Apply the revisions in topological order such that a revision
# is only written once it's deltabase and parents have been written.
while queue:
f, node = queue.pop(0)
if (f, node) in processed:
continue
skipcount += 1
if skipcount > len(queue) + 1:
raise error.Abort(_(b"circular node dependency"))
fl = repo.file(f)
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags, sdata, pfl)
(
node,
p1,
p2,
linknode,
deltabase,
delta,
flags,
sidedata,
proto_flags,
) = revisiondata
if not available(f, node, f, deltabase):
continue
base = fl.rawdata(deltabase)
text = mdiff.patch(base, delta)
if not isinstance(text, bytes):
text = bytes(text)
meta, text = shallowutil.parsemeta(text)
if b'copy' in meta:
copyfrom = meta[b'copy']
copynode = bin(meta[b'copyrev'])
if not available(f, node, copyfrom, copynode):
continue
for p in [p1, p2]:
if p != repo.nullid:
if not available(f, node, f, p):
continue
fl.add(text, meta, trp, linknode, p1, p2)
processed.add((f, node))
skipcount = 0
progress.complete()
return len(revisiondatas), newfiles