treemanifests: fix streaming clone

treemanifests: fix streaming clone

Similar to the previous patch, the .hg/store/meta/ directory does not
get copied when when using "hg clone --uncompressed". Fix by including
"meta/" in store.datafiles(). This seems safe to do, as there are only
a few users of this method. "hg manifest" already filters the paths by
"data/" prefix. The calls from largefiles also seem safe. The use in
verify needs updating to prevent it from mistaking dirlogs for
orphaned filelogs. That change is included in this patch.

Since the dirlogs will now be in the fncache when using fncachestore,
let's also update debugrebuildfncache(). That will also allow any
existing treemanifest repos to get their dirlogs into the fncache.

Also update test-treemanifest.t to use an a directory name that
requires dot-encoding and uppercase-encoding so we test that the path
encoding works.

--- a/mercurial/repair.py	Tue Feb 02 17:31:17 2016 -0800
+++ b/mercurial/repair.py	Thu Feb 04 08:34:07 2016 -0800
@@ -273,6 +273,16 @@
 
         ui.progress(_('changeset'), None)
 
+        if 'treemanifest' in repo: # safe but unnecessary otherwise
+            for dir in util.dirs(seenfiles):
+                i = 'meta/%s/00manifest.i' % dir
+                d = 'meta/%s/00manifest.d' % dir
+
+                if repo.store._exists(i):
+                    newentries.add(i)
+                if repo.store._exists(d):
+                    newentries.add(d)
+
         addcount = len(newentries - oldentries)
         removecount = len(oldentries - newentries)
         for p in sorted(oldentries - newentries):

--- a/mercurial/store.py	Tue Feb 02 17:31:17 2016 -0800
+++ b/mercurial/store.py	Thu Feb 04 08:34:07 2016 -0800
@@ -330,7 +330,7 @@
         return l
 
     def datafiles(self):
-        return self._walk('data', True)
+        return self._walk('data', True) + self._walk('meta', True)
 
     def topfiles(self):
         # yield manifest before changelog
@@ -378,7 +378,7 @@
         self.opener = self.vfs
 
     def datafiles(self):
-        for a, b, size in self._walk('data', True):
+        for a, b, size in super(encodedstore, self).datafiles():
             try:
                 a = decodefilename(a)
             except KeyError:
@@ -460,7 +460,8 @@
         self.encode = encode
 
     def __call__(self, path, mode='r', *args, **kw):
-        if mode not in ('r', 'rb') and path.startswith('data/'):
+        if mode not in ('r', 'rb') and (path.startswith('data/') or
+                                        path.startswith('meta/')):
             self.fncache.add(path)
         return self.vfs(self.encode(path), mode, *args, **kw)
 

--- a/mercurial/verify.py	Tue Feb 02 17:31:17 2016 -0800
+++ b/mercurial/verify.py	Thu Feb 04 08:34:07 2016 -0800
@@ -284,7 +284,7 @@
         for f, f2, size in repo.store.datafiles():
             if not f:
                 self.err(None, _("cannot decode filename '%s'") % f2)
-            elif size > 0 or not revlogv1:
+            elif (size > 0 or not revlogv1) and f.startswith('data/'):
                 storefiles.add(_normpath(f))
 
         files = sorted(set(filenodes) | set(filelinkrevs))

--- a/tests/run-tests.py	Tue Feb 02 17:31:17 2016 -0800
+++ b/tests/run-tests.py	Thu Feb 04 08:34:07 2016 -0800
@@ -719,6 +719,8 @@
             (br':%d\b' % self._startport, b':$HGPORT'),
             (br':%d\b' % (self._startport + 1), b':$HGPORT1'),
             (br':%d\b' % (self._startport + 2), b':$HGPORT2'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT3'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT4'),
             (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
              br'\1 (glob)'),
             ]
@@ -741,6 +743,8 @@
         env["HGPORT"] = str(self._startport)
         env["HGPORT1"] = str(self._startport + 1)
         env["HGPORT2"] = str(self._startport + 2)
+        env["HGPORT3"] = str(self._startport + 3)
+        env["HGPORT4"] = str(self._startport + 4)
         env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc')
         env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids')
         env["HGEDITOR"] = ('"' + sys.executable + '"'

--- a/tests/test-treemanifest.t	Tue Feb 02 17:31:17 2016 -0800
+++ b/tests/test-treemanifest.t	Thu Feb 04 08:34:07 2016 -0800
@@ -367,7 +367,7 @@
   $ hg --config experimental.treemanifest=True init deeprepo
   $ cd deeprepo
 
-  $ mkdir a
+  $ mkdir .A
   $ mkdir b
   $ mkdir b/bar
   $ mkdir b/bar/orange
@@ -376,8 +376,8 @@
   $ mkdir b/foo/apple
   $ mkdir b/foo/apple/bees
 
-  $ touch a/one.txt
-  $ touch a/two.txt
+  $ touch .A/one.txt
+  $ touch .A/two.txt
   $ touch b/bar/fruits.txt
   $ touch b/bar/orange/fly/gnat.py
   $ touch b/bar/orange/fly/housefly.txt
@@ -393,8 +393,8 @@
 Test files from the root.
 
   $ hg files -r .
-  a/one.txt (glob)
-  a/two.txt (glob)
+  .A/one.txt (glob)
+  .A/two.txt (glob)
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
@@ -412,7 +412,7 @@
 
 Test files for a subdirectory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ hg files -r . b
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
@@ -422,7 +422,7 @@
 
 Test files with just includes and excludes.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/bar/orange/fly
   $ rm -r .hg/store/meta/b/foo/apple/bees
   $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees
@@ -431,7 +431,7 @@
 
 Test files for a subdirectory, excluding a directory within it.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -X path:b/foo b
   b/bar/fruits.txt (glob)
@@ -442,7 +442,7 @@
 Test files for a sub directory, including only a directory within it, and
 including an unrelated directory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -I path:b/bar/orange -I path:a b
   b/bar/orange/fly/gnat.py (glob)
@@ -452,7 +452,7 @@
 Test files for a pattern, including a directory, and excluding a directory
 within that.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ rm -r .hg/store/meta/b/bar/orange
   $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange
@@ -488,8 +488,6 @@
 Tree manifest revlogs exist.
   $ find deepclone/.hg/store/meta | sort
   deepclone/.hg/store/meta
-  deepclone/.hg/store/meta/a
-  deepclone/.hg/store/meta/a/00manifest.i
   deepclone/.hg/store/meta/b
   deepclone/.hg/store/meta/b/00manifest.i
   deepclone/.hg/store/meta/b/bar
@@ -504,6 +502,8 @@
   deepclone/.hg/store/meta/b/foo/apple/00manifest.i
   deepclone/.hg/store/meta/b/foo/apple/bees
   deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i
+  deepclone/.hg/store/meta/~2e_a
+  deepclone/.hg/store/meta/~2e_a/00manifest.i
 Verify passes.
   $ cd deepclone
   $ hg verify
@@ -525,8 +525,12 @@
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep store deeprepo-basicstore/.hg/requires
+  $ cd deeprepo-basicstore
+  $ grep store .hg/requires
   [1]
+  $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
   $ hg clone --config format.usefncache=False \
   >   --config experimental.changegroup3=True \
   >   http://localhost:$HGPORT2 deeprepo-encodedstore
@@ -537,8 +541,12 @@
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep fncache deeprepo-encodedstore/.hg/requires
+  $ cd deeprepo-encodedstore
+  $ grep fncache .hg/requires
   [1]
+  $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
 
 Local clone with basicstore
   $ hg clone -U deeprepo-basicstore local-clone-basicstore
@@ -566,3 +574,55 @@
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
+
+Stream clone with basicstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT3 stream-clone-basicstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-basicstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with encodedstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT4 stream-clone-encodedstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-encodedstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with fncachestore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT2 stream-clone-fncachestore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-fncachestore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Packed bundle
+  $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg
+  writing 3349 bytes for 18 files
+  bundle requirements: generaldelta, revlogv1, treemanifest
+  $ hg debugbundle --spec repo-packed.hg
+  none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest