Mercurial > hg
changeset 28007:fb92927f9775
treemanifests: fix streaming clone
Similar to the previous patch, the .hg/store/meta/ directory does not
get copied when when using "hg clone --uncompressed". Fix by including
"meta/" in store.datafiles(). This seems safe to do, as there are only
a few users of this method. "hg manifest" already filters the paths by
"data/" prefix. The calls from largefiles also seem safe. The use in
verify needs updating to prevent it from mistaking dirlogs for
orphaned filelogs. That change is included in this patch.
Since the dirlogs will now be in the fncache when using fncachestore,
let's also update debugrebuildfncache(). That will also allow any
existing treemanifest repos to get their dirlogs into the fncache.
Also update test-treemanifest.t to use an a directory name that
requires dot-encoding and uppercase-encoding so we test that the path
encoding works.
| author | Martin von Zweigbergk <martinvonz@google.com> |
|---|---|
| date | Thu, 04 Feb 2016 08:34:07 -0800 |
| parents | bb45190a54b7 |
| children | 86c4cbdaffee |
| files | mercurial/repair.py mercurial/store.py mercurial/verify.py tests/run-tests.py tests/test-treemanifest.t |
| diffstat | 5 files changed, 93 insertions(+), 18 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/repair.py Tue Feb 02 17:31:17 2016 -0800 +++ b/mercurial/repair.py Thu Feb 04 08:34:07 2016 -0800 @@ -273,6 +273,16 @@ ui.progress(_('changeset'), None) + if 'treemanifest' in repo: # safe but unnecessary otherwise + for dir in util.dirs(seenfiles): + i = 'meta/%s/00manifest.i' % dir + d = 'meta/%s/00manifest.d' % dir + + if repo.store._exists(i): + newentries.add(i) + if repo.store._exists(d): + newentries.add(d) + addcount = len(newentries - oldentries) removecount = len(oldentries - newentries) for p in sorted(oldentries - newentries):
--- a/mercurial/store.py Tue Feb 02 17:31:17 2016 -0800 +++ b/mercurial/store.py Thu Feb 04 08:34:07 2016 -0800 @@ -330,7 +330,7 @@ return l def datafiles(self): - return self._walk('data', True) + return self._walk('data', True) + self._walk('meta', True) def topfiles(self): # yield manifest before changelog @@ -378,7 +378,7 @@ self.opener = self.vfs def datafiles(self): - for a, b, size in self._walk('data', True): + for a, b, size in super(encodedstore, self).datafiles(): try: a = decodefilename(a) except KeyError: @@ -460,7 +460,8 @@ self.encode = encode def __call__(self, path, mode='r', *args, **kw): - if mode not in ('r', 'rb') and path.startswith('data/'): + if mode not in ('r', 'rb') and (path.startswith('data/') or + path.startswith('meta/')): self.fncache.add(path) return self.vfs(self.encode(path), mode, *args, **kw)
--- a/mercurial/verify.py Tue Feb 02 17:31:17 2016 -0800 +++ b/mercurial/verify.py Thu Feb 04 08:34:07 2016 -0800 @@ -284,7 +284,7 @@ for f, f2, size in repo.store.datafiles(): if not f: self.err(None, _("cannot decode filename '%s'") % f2) - elif size > 0 or not revlogv1: + elif (size > 0 or not revlogv1) and f.startswith('data/'): storefiles.add(_normpath(f)) files = sorted(set(filenodes) | set(filelinkrevs))
--- a/tests/run-tests.py Tue Feb 02 17:31:17 2016 -0800 +++ b/tests/run-tests.py Thu Feb 04 08:34:07 2016 -0800 @@ -719,6 +719,8 @@ (br':%d\b' % self._startport, b':$HGPORT'), (br':%d\b' % (self._startport + 1), b':$HGPORT1'), (br':%d\b' % (self._startport + 2), b':$HGPORT2'), + (br':%d\b' % (self._startport + 2), b':$HGPORT3'), + (br':%d\b' % (self._startport + 2), b':$HGPORT4'), (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$', br'\1 (glob)'), ] @@ -741,6 +743,8 @@ env["HGPORT"] = str(self._startport) env["HGPORT1"] = str(self._startport + 1) env["HGPORT2"] = str(self._startport + 2) + env["HGPORT3"] = str(self._startport + 3) + env["HGPORT4"] = str(self._startport + 4) env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc') env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids') env["HGEDITOR"] = ('"' + sys.executable + '"'
--- a/tests/test-treemanifest.t Tue Feb 02 17:31:17 2016 -0800 +++ b/tests/test-treemanifest.t Thu Feb 04 08:34:07 2016 -0800 @@ -367,7 +367,7 @@ $ hg --config experimental.treemanifest=True init deeprepo $ cd deeprepo - $ mkdir a + $ mkdir .A $ mkdir b $ mkdir b/bar $ mkdir b/bar/orange @@ -376,8 +376,8 @@ $ mkdir b/foo/apple $ mkdir b/foo/apple/bees - $ touch a/one.txt - $ touch a/two.txt + $ touch .A/one.txt + $ touch .A/two.txt $ touch b/bar/fruits.txt $ touch b/bar/orange/fly/gnat.py $ touch b/bar/orange/fly/housefly.txt @@ -393,8 +393,8 @@ Test files from the root. $ hg files -r . - a/one.txt (glob) - a/two.txt (glob) + .A/one.txt (glob) + .A/two.txt (glob) b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) @@ -412,7 +412,7 @@ Test files for a subdirectory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ hg files -r . b b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) @@ -422,7 +422,7 @@ Test files with just includes and excludes. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/bar/orange/fly $ rm -r .hg/store/meta/b/foo/apple/bees $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees @@ -431,7 +431,7 @@ Test files for a subdirectory, excluding a directory within it. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -X path:b/foo b b/bar/fruits.txt (glob) @@ -442,7 +442,7 @@ Test files for a sub directory, including only a directory within it, and including an unrelated directory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -I path:b/bar/orange -I path:a b b/bar/orange/fly/gnat.py (glob) @@ -452,7 +452,7 @@ Test files for a pattern, including a directory, and excluding a directory within that. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ rm -r .hg/store/meta/b/bar/orange $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange @@ -488,8 +488,6 @@ Tree manifest revlogs exist. $ find deepclone/.hg/store/meta | sort deepclone/.hg/store/meta - deepclone/.hg/store/meta/a - deepclone/.hg/store/meta/a/00manifest.i deepclone/.hg/store/meta/b deepclone/.hg/store/meta/b/00manifest.i deepclone/.hg/store/meta/b/bar @@ -504,6 +502,8 @@ deepclone/.hg/store/meta/b/foo/apple/00manifest.i deepclone/.hg/store/meta/b/foo/apple/bees deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i + deepclone/.hg/store/meta/~2e_a + deepclone/.hg/store/meta/~2e_a/00manifest.i Verify passes. $ cd deepclone $ hg verify @@ -525,8 +525,12 @@ added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep store deeprepo-basicstore/.hg/requires + $ cd deeprepo-basicstore + $ grep store .hg/requires [1] + $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. $ hg clone --config format.usefncache=False \ > --config experimental.changegroup3=True \ > http://localhost:$HGPORT2 deeprepo-encodedstore @@ -537,8 +541,12 @@ added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep fncache deeprepo-encodedstore/.hg/requires + $ cd deeprepo-encodedstore + $ grep fncache .hg/requires [1] + $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. Local clone with basicstore $ hg clone -U deeprepo-basicstore local-clone-basicstore @@ -566,3 +574,55 @@ crosschecking files in changesets and manifests checking files 8 files, 3 changesets, 10 total revisions + +Stream clone with basicstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT3 stream-clone-basicstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-basicstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with encodedstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT4 stream-clone-encodedstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-encodedstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with fncachestore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT2 stream-clone-fncachestore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-fncachestore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Packed bundle + $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg + writing 3349 bytes for 18 files + bundle requirements: generaldelta, revlogv1, treemanifest + $ hg debugbundle --spec repo-packed.hg + none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest