narrow: detect if narrowspec was changed in a different share

narrow: detect if narrowspec was changed in a different share

With this commit, `hg share` should be usable with narrow
repos. Design explained on
https://www.mercurial-scm.org/wiki/NarrowSharePlan

I was running into cache invalidation problems when updating the
narrowspec. After spending a day trying to figure out a good solution,
I resorted to just assigning repo.narrowpats and repo._narrowmatch
after invalidating them.

Differential Revision: https://phab.mercurial-scm.org/D5278

--- a/hgext/narrow/TODO.rst	Fri Jul 13 11:26:46 2018 -0700
+++ b/hgext/narrow/TODO.rst	Fri Dec 21 10:13:49 2018 -0800
@@ -1,6 +1,3 @@
-Integration with the share extension needs improvement. Right now
-we've seen some odd bugs.
-
 Address commentary in manifest.excludedmanifestrevlog.add -
 specifically we should improve the collaboration with core so that
 add() never gets called on an excluded directory and we can improve

--- a/hgext/narrow/narrowcommands.py	Fri Jul 13 11:26:46 2018 -0700
+++ b/hgext/narrow/narrowcommands.py	Fri Dec 21 10:13:49 2018 -0800
@@ -339,6 +339,8 @@
      ('', 'clear', False, _('whether to replace the existing narrowspec')),
      ('', 'force-delete-local-changes', False,
        _('forces deletion of local changes when narrowing')),
+     ('', 'update-working-copy', False,
+      _('update working copy when the store has changed')),
     ] + commands.remoteopts,
     _('[OPTIONS]... [REMOTE]'),
     inferrepo=True)
@@ -398,8 +400,9 @@
     addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
     removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
 
+    update_working_copy = opts['update_working_copy']
     only_show = not (addedincludes or removedincludes or addedexcludes or
-                     removedexcludes or newrules)
+                     removedexcludes or newrules or update_working_copy)
 
     oldincludes, oldexcludes = repo.narrowpats
 
@@ -428,6 +431,12 @@
         fm.end()
         return 0
 
+    if update_working_copy:
+        with repo.wlock(), repo.lock(), repo.transaction('narrow-wc') as tr:
+            narrowspec.updateworkingcopy(repo, tr)
+            narrowspec.copytoworkingcopy(repo, tr)
+        return 0
+
     if not widening and not narrowing:
         ui.status(_("nothing to widen or narrow\n"))
         return 0

--- a/mercurial/hg.py	Fri Jul 13 11:26:46 2018 -0700
+++ b/mercurial/hg.py	Fri Dec 21 10:13:49 2018 -0800
@@ -38,6 +38,7 @@
     narrowspec,
     node,
     phases,
+    repository as repositorymod,
     scmutil,
     sshpeer,
     statichttprepo,
@@ -331,6 +332,9 @@
         template = ('[paths]\n'
                     'default = %s\n')
         destrepo.vfs.write('hgrc', util.tonativeeol(template % default))
+    if repositorymod.NARROW_REQUIREMENT in sourcerepo.requirements:
+        with destrepo.wlock():
+            narrowspec.copytoworkingcopy(destrepo, None)
 
 def _postshareupdate(repo, update, checkout=None):
     """Maybe perform a working directory update after a shared repo is created.
@@ -731,7 +735,7 @@
             local = destpeer.local()
             if local:
                 if narrow:
-                    with local.lock():
+                    with local.wlock(), local.lock():
                         local.setnarrowpats(storeincludepats, storeexcludepats)
 
                 u = util.url(abspath)

--- a/mercurial/localrepo.py	Fri Jul 13 11:26:46 2018 -0700
+++ b/mercurial/localrepo.py	Fri Dec 21 10:13:49 2018 -0800
@@ -1227,6 +1227,7 @@
     def _narrowmatch(self):
         if repository.NARROW_REQUIREMENT not in self.requirements:
             return matchmod.always(self.root, '')
+        narrowspec.checkworkingcopynarrowspec(self)
         include, exclude = self.narrowpats
         return narrowspec.match(self.root, include=include, exclude=exclude)
 
@@ -1251,7 +1252,14 @@
 
     def setnarrowpats(self, newincludes, newexcludes):
         narrowspec.save(self, newincludes, newexcludes)
+        narrowspec.copytoworkingcopy(self, self.currenttransaction())
         self.invalidate(clearfilecache=True)
+        # So the next access won't be considered a conflict
+        # TODO: It seems like there should be a way of doing this that
+        # doesn't involve replacing these attributes.
+        self.narrowpats = newincludes, newexcludes
+        self._narrowmatch = narrowspec.match(self.root, include=newincludes,
+                                             exclude=newexcludes)
 
     def __getitem__(self, changeid):
         if changeid is None:

--- a/mercurial/narrowspec.py	Fri Jul 13 11:26:46 2018 -0700
+++ b/mercurial/narrowspec.py	Fri Dec 21 10:13:49 2018 -0800
@@ -13,12 +13,16 @@
 from . import (
     error,
     match as matchmod,
+    merge,
     repository,
     sparse,
     util,
 )
 
+# The file in .hg/store/ that indicates which paths exit in the store
 FILENAME = 'narrowspec'
+# The file in .hg/ that indicates which paths exit in the dirstate
+DIRSTATE_FILENAME = 'narrowspec.dirstate'
 
 # Pattern prefixes that are allowed in narrow patterns. This list MUST
 # only contain patterns that are fast and safe to evaluate. Keep in mind
@@ -157,6 +161,18 @@
     spec = format(includepats, excludepats)
     repo.svfs.write(FILENAME, spec)
 
+def copytoworkingcopy(repo, tr):
+    if tr:
+        def write(file):
+            spec = repo.svfs.read(FILENAME)
+            file.write(spec)
+            file.close()
+        tr.addfilegenerator('narrowspec', (DIRSTATE_FILENAME,), write,
+                            location='plain')
+    else:
+        spec = repo.svfs.read(FILENAME)
+        repo.vfs.write(DIRSTATE_FILENAME, spec)
+
 def savebackup(repo, backupname):
     if repository.NARROW_REQUIREMENT not in repo.requirements:
         return
@@ -226,3 +242,57 @@
     else:
         res_includes = set(req_includes)
     return res_includes, res_excludes, invalid_includes
+
+# These two are extracted for extensions (specifically for Google's CitC file
+# system)
+def _deletecleanfiles(repo, files):
+    for f in files:
+        repo.wvfs.unlinkpath(f)
+
+def _writeaddedfiles(repo, pctx, files):
+    actions = merge.emptyactions()
+    addgaction = actions['g'].append
+    mf = repo['.'].manifest()
+    for f in files:
+        if not repo.wvfs.exists(f):
+            addgaction((f, (mf.flags(f), False), "narrowspec updated"))
+    merge.applyupdates(repo, actions, wctx=repo[None],
+                       mctx=repo['.'], overwrite=False)
+
+def checkworkingcopynarrowspec(repo):
+    storespec = repo.svfs.tryread(FILENAME)
+    wcspec = repo.vfs.tryread(DIRSTATE_FILENAME)
+    if wcspec != storespec:
+        raise error.Abort(_("working copy's narrowspec is stale"),
+                          hint=_("run 'hg tracked --update-working-copy'"))
+
+def updateworkingcopy(repo, tr):
+    oldspec = repo.vfs.tryread(DIRSTATE_FILENAME)
+    newspec = repo.svfs.tryread(FILENAME)
+
+    oldincludes, oldexcludes = parseconfig(repo.ui, oldspec)
+    newincludes, newexcludes = parseconfig(repo.ui, newspec)
+    oldmatch = match(repo.root, include=oldincludes, exclude=oldexcludes)
+    newmatch = match(repo.root, include=newincludes, exclude=newexcludes)
+    addedmatch = matchmod.differencematcher(newmatch, oldmatch)
+    removedmatch = matchmod.differencematcher(oldmatch, newmatch)
+
+    ds = repo.dirstate
+    lookup, status = ds.status(removedmatch, subrepos=[], ignored=False,
+                               clean=True, unknown=False)
+    _deletecleanfiles(repo, status.clean)
+    trackeddirty = lookup + status.modified + status.added
+    for f in sorted(trackeddirty):
+        repo.ui.status(_('not deleting possibly dirty file %s\n') % f)
+    for f in status.clean + trackeddirty:
+        ds.drop(f)
+
+    repo.narrowpats = newincludes, newexcludes
+    repo._narrowmatch = newmatch
+    pctx = repo['.']
+    newfiles = [f for f in pctx.manifest().walk(addedmatch) if f not in ds]
+    for f in newfiles:
+        ds.normallookup(f)
+    _writeaddedfiles(repo, pctx, newfiles)
+
+    ds.write(tr)

--- a/tests/test-narrow-debugcommands.t	Fri Jul 13 11:26:46 2018 -0700
+++ b/tests/test-narrow-debugcommands.t	Fri Dec 21 10:13:49 2018 -0800
@@ -6,6 +6,7 @@
   > path:foo
   > [exclude]
   > EOF
+  $ cp .hg/store/narrowspec .hg/narrowspec.dirstate
   $ echo treemanifest >> .hg/requires
   $ echo narrowhg-experimental >> .hg/requires
   $ mkdir -p foo/bar

--- a/tests/test-narrow-share.t	Fri Jul 13 11:26:46 2018 -0700
+++ b/tests/test-narrow-share.t	Fri Dec 21 10:13:49 2018 -0800
@@ -75,13 +75,20 @@
   deleting meta/d5/00manifest.i (tree !)
   $ hg -R main tracked
   I path:d7
+  $ hg -R main files
+  abort: working copy's narrowspec is stale
+  (run 'hg tracked --update-working-copy')
+  [255]
+  $ hg -R main tracked --update-working-copy
+  not deleting possibly dirty file d3/f
+  not deleting possibly dirty file d3/g
+  not deleting possibly dirty file d5/f
 # d1/f, d3/f, d3/g and d5/f should no longer be reported
   $ hg -R main files
   main/d7/f
 # d1/f should no longer be there, d3/f should be since it was dirty, d3/g should be there since
 # it was added, and d5/f should be since we couldn't be sure it was clean
   $ find main/d* -type f | sort
-  main/d1/f
   main/d3/f
   main/d3/g
   main/d5/f
@@ -102,16 +109,20 @@
   I path:d1
   I path:d3
   I path:d7
+  $ hg -R main files
+  abort: working copy's narrowspec is stale
+  (run 'hg tracked --update-working-copy')
+  [255]
+  $ hg -R main tracked --update-working-copy
 # d1/f, d3/f should be back
   $ hg -R main files
   main/d1/f
   main/d3/f
-  main/d3/g
   main/d7/f
 # d3/f should be modified (not clobbered by the widening), and d3/g should be untracked
   $ hg -R main st --all
   M d3/f
-  A d3/g
+  ? d3/g
   C d1/f
   C d7/f
 
@@ -130,3 +141,30 @@
   checking files
   checked 11 changesets with 3 changes to 3 files
   $ cd ..
+
+Dirstate should be left alone when upgrading from version of hg that didn't support narrow+share
+
+  $ hg share main share-upgrade
+  updating working directory
+  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd share-upgrade
+  $ echo x >> d1/f
+  $ echo y >> d3/g
+  $ hg add d3/g
+  $ hg rm d7/f
+  $ hg st
+  M d1/f
+  A d3/g
+  R d7/f
+Make it look like a repo from before narrow+share was supported
+  $ rm .hg/narrowspec.dirstate
+  $ hg st
+  abort: working copy's narrowspec is stale
+  (run 'hg tracked --update-working-copy')
+  [255]
+  $ hg tracked --update-working-copy
+  $ hg st
+  M d1/f
+  A d3/g
+  R d7/f
+  $ cd ..

--- a/tests/test-narrow-trackedcmd.t	Fri Jul 13 11:26:46 2018 -0700
+++ b/tests/test-narrow-trackedcmd.t	Fri Dec 21 10:13:49 2018 -0800
@@ -107,6 +107,8 @@
       --clear                      whether to replace the existing narrowspec
       --force-delete-local-changes forces deletion of local changes when
                                    narrowing
+      --update-working-copy        update working copy when the store has
+                                   changed
    -e --ssh CMD                    specify ssh command to use
       --remotecmd CMD              specify hg command to run on the remote side
       --insecure                   do not verify server certificate (ignoring