changeset 33320:153456f02426

sparse: move function for resolving sparse matcher into core

As part of the move, the function arguments changed so revs are
passed as a list instead of *args. This allows us to use keyword
arguments properly.

Since the plan is to integrate sparse into core and have it
enabled by default, we need to prepare for a sparse matcher
to always be obtained and operated on. As part of the move,
we inserted code that returns an always matcher if sparse
isn't enabled. Some callers in the sparse extension take this
into account and conditionally perform matching depending on
whether the special always matcher is seen. I /think/ this
may have sped up some operations where the extension is
installed but no sparse config is activated.

One thing I'm ensure of in this code is whether os.path.dirname()
is semantically correct. os.posixpath.dirname() (which is
exported as pathutil.dirname) might be a better choise because
all patterns should be using posix directory separators (/)
instead of Windows (\). There's an inline comment that implies
Windows was tested. So hopefully it won't be a problem. We
can improve this in a follow-up. I've added a TODO to track it.

--- a/hgext/sparse.py	Thu Jul 06 17:39:24 2017 -0700
+++ b/hgext/sparse.py	Thu Jul 06 17:41:45 2017 -0700
@@ -75,7 +75,6 @@
 from __future__ import absolute_import
 
 import collections
-import os
 
 from mercurial.i18n import _
 from mercurial.node import nullid
@@ -146,22 +145,23 @@
         actions, diverge, renamedelete = orig(repo, wctx, mctx, ancestors,
                                               branchmerge, *arg, **kwargs)
 
-        if not util.safehasattr(repo, 'sparsematch'):
+        oldrevs = [pctx.rev() for pctx in wctx.parents()]
+        oldsparsematch = sparse.matcher(repo, oldrevs)
+
+        if oldsparsematch.always():
             return actions, diverge, renamedelete
 
         files = set()
         prunedactions = {}
-        oldrevs = [pctx.rev() for pctx in wctx.parents()]
-        oldsparsematch = repo.sparsematch(*oldrevs)
 
         if branchmerge:
             # If we're merging, use the wctx filter, since we're merging into
             # the wctx.
-            sparsematch = repo.sparsematch(wctx.parents()[0].rev())
+            sparsematch = sparse.matcher(repo, [wctx.parents()[0].rev()])
         else:
             # If we're updating, use the target context's filter, since we're
             # moving to the target context.
-            sparsematch = repo.sparsematch(mctx.rev())
+            sparsematch = sparse.matcher(repo, [mctx.rev()])
 
         temporaryfiles = []
         for file, action in actions.iteritems():
@@ -229,7 +229,7 @@
 
         # If we're updating to a location, clean up any stale temporary includes
         # (ex: this happens during hg rebase --abort).
-        if not branchmerge and util.safehasattr(repo, 'sparsematch'):
+        if not branchmerge and util.safehasattr(repo, 'prunetemporaryincludes'):
             repo.prunetemporaryincludes()
         return results
 
@@ -248,7 +248,7 @@
         # profiles will only have data if sparse is enabled.
         if set(profiles) & set(ctx.files()):
             origstatus = repo.status()
-            origsparsematch = repo.sparsematch()
+            origsparsematch = sparse.matcher(repo)
             _refresh(repo.ui, repo, origstatus, origsparsematch, True)
 
         if util.safehasattr(repo, 'prunetemporaryincludes'):
@@ -265,7 +265,7 @@
     def _logrevs(orig, repo, opts):
         revs = orig(repo, opts)
         if opts.get('sparse'):
-            sparsematch = repo.sparsematch()
+            sparsematch = sparse.matcher(repo)
             def ctxmatch(rev):
                 ctx = repo[rev]
                 return any(f for f in ctx.files() if sparsematch(f))
@@ -347,10 +347,11 @@
         def __get__(self, obj, type=None):
             repo = obj.repo
             origignore = self.orig.__get__(obj)
-            if not util.safehasattr(repo, 'sparsematch'):
+
+            sparsematch = sparse.matcher(repo)
+            if sparsematch.always():
                 return origignore
 
-            sparsematch = repo.sparsematch()
             if self.sparsematch != sparsematch or self.origignore != origignore:
                 self.func = matchmod.unionmatcher([
                     origignore, matchmod.negatematcher(sparsematch)])
@@ -368,8 +369,8 @@
 
     # dirstate.rebuild should not add non-matching files
     def _rebuild(orig, self, parent, allfiles, changedfiles=None):
-        if util.safehasattr(self.repo, 'sparsematch'):
-            matcher = self.repo.sparsematch()
+        matcher = sparse.matcher(self.repo)
+        if not matcher.always():
             allfiles = allfiles.matches(matcher)
             if changedfiles:
                 changedfiles = [f for f in changedfiles if matcher(f)]
@@ -390,9 +391,9 @@
     for func in editfuncs:
         def _wrapper(orig, self, *args):
             repo = self.repo
-            if util.safehasattr(repo, 'sparsematch'):
+            sparsematch = sparse.matcher(repo)
+            if not sparsematch.always():
                 dirstate = repo.dirstate
-                sparsematch = repo.sparsematch()
                 for f in args:
                     if (f is not None and not sparsematch(f) and
                         f not in dirstate):
@@ -404,72 +405,6 @@
 
 def _wraprepo(ui, repo):
     class SparseRepo(repo.__class__):
-        def sparsematch(self, *revs, **kwargs):
-            """Returns the sparse match function for the given revs.
-
-            If multiple revs are specified, the match function is the union
-            of all the revs.
-
-            `includetemp` is used to indicate if the temporarily included file
-            should be part of the matcher.
-            """
-            if not revs or revs == (None,):
-                revs = [self.changelog.rev(node) for node in
-                    self.dirstate.parents() if node != nullid]
-
-            includetemp = kwargs.get('includetemp', True)
-            signature = sparse.configsignature(self, includetemp=includetemp)
-
-            key = '%s %s' % (str(signature), ' '.join([str(r) for r in revs]))
-
-            result = self._sparsematchercache.get(key, None)
-            if result:
-                return result
-
-            matchers = []
-            for rev in revs:
-                try:
-                    includes, excludes, profiles = sparse.patternsforrev(
-                        self, rev)
-
-                    if includes or excludes:
-                        # Explicitly include subdirectories of includes so
-                        # status will walk them down to the actual include.
-                        subdirs = set()
-                        for include in includes:
-                            dirname = os.path.dirname(include)
-                            # basename is used to avoid issues with absolute
-                            # paths (which on Windows can include the drive).
-                            while os.path.basename(dirname):
-                                subdirs.add(dirname)
-                                dirname = os.path.dirname(dirname)
-
-                        matcher = matchmod.match(self.root, '', [],
-                            include=includes, exclude=excludes,
-                            default='relpath')
-                        if subdirs:
-                            matcher = matchmod.forceincludematcher(matcher,
-                                                                   subdirs)
-                        matchers.append(matcher)
-                except IOError:
-                    pass
-
-            result = None
-            if not matchers:
-                result = matchmod.always(self.root, '')
-            elif len(matchers) == 1:
-                result = matchers[0]
-            else:
-                result = matchmod.unionmatcher(matchers)
-
-            if kwargs.get('includetemp', True):
-                tempincludes = sparse.readtemporaryincludes(self)
-                result = matchmod.forceincludematcher(result, tempincludes)
-
-            self._sparsematchercache[key] = result
-
-            return result
-
         def prunetemporaryincludes(self):
             if repo.vfs.exists('tempsparse'):
                 origstatus = self.status()
@@ -478,7 +413,7 @@
                     # Still have pending changes. Don't bother trying to prune.
                     return
 
-                sparsematch = self.sparsematch(includetemp=False)
+                sparsematch = sparse.matcher(self, includetemp=False)
                 dirstate = self.dirstate
                 actions = []
                 dropped = []
@@ -605,7 +540,7 @@
             wlock = repo.wlock()
             fcounts = map(
                 len,
-                _refresh(ui, repo, repo.status(), repo.sparsematch(), force))
+                _refresh(ui, repo, repo.status(), sparse.matcher(repo), force))
             _verbose_output(ui, opts, 0, 0, 0, *fcounts)
         finally:
             wlock.release()
@@ -618,7 +553,7 @@
     """
     wlock = repo.wlock()
     try:
-        oldsparsematch = repo.sparsematch()
+        oldsparsematch = sparse.matcher(repo)
 
         raw = repo.vfs.tryread('sparse')
         if raw:
@@ -718,7 +653,7 @@
             excludecount = len(excludes - aexcludes)
 
             oldstatus = repo.status()
-            oldsparsematch = repo.sparsematch()
+            oldsparsematch = sparse.matcher(repo)
             sparse.writeconfig(repo, includes, excludes, profiles)
 
             try:
@@ -738,7 +673,7 @@
 
         if includes or excludes:
             oldstatus = repo.status()
-            oldsparsematch = repo.sparsematch()
+            oldsparsematch = sparse.matcher(repo)
             sparse.writeconfig(repo, set(), set(), profiles)
             _refresh(ui, repo, oldstatus, oldsparsematch, force)
 
@@ -756,7 +691,7 @@
     pending.update(modified)
     pending.update(added)
     pending.update(removed)
-    sparsematch = repo.sparsematch()
+    sparsematch = sparse.matcher(repo)
     abort = False
     for file in pending:
         if not sparsematch(file):

--- a/mercurial/sparse.py	Thu Jul 06 17:39:24 2017 -0700
+++ b/mercurial/sparse.py	Thu Jul 06 17:41:45 2017 -0700
@@ -8,11 +8,14 @@
 from __future__ import absolute_import
 
 import hashlib
+import os
 
 from .i18n import _
 from .node import nullid
 from . import (
     error,
+    match as matchmod,
+    pycompat,
 )
 
 # Whether sparse features are enabled. This variable is intended to be
@@ -193,3 +196,70 @@
     for i in additional:
         includes.add(i)
     writetemporaryincludes(repo, includes)
+
+def matcher(repo, revs=None, includetemp=True):
+    """Obtain a matcher for sparse working directories for the given revs.
+
+    If multiple revisions are specified, the matcher is the union of all
+    revs.
+
+    ``includetemp`` indicates whether to use the temporary sparse profile.
+    """
+    # If sparse isn't enabled, sparse matcher matches everything.
+    if not enabled:
+        return matchmod.always(repo.root, '')
+
+    if not revs or revs == [None]:
+        revs = [repo.changelog.rev(node)
+                for node in repo.dirstate.parents() if node != nullid]
+
+    signature = configsignature(repo, includetemp=includetemp)
+
+    key = '%s %s' % (signature, ' '.join(map(pycompat.bytestr, revs)))
+
+    result = repo._sparsematchercache.get(key)
+    if result:
+        return result
+
+    matchers = []
+    for rev in revs:
+        try:
+            includes, excludes, profiles = patternsforrev(repo, rev)
+
+            if includes or excludes:
+                # Explicitly include subdirectories of includes so
+                # status will walk them down to the actual include.
+                subdirs = set()
+                for include in includes:
+                    # TODO consider using posix path functions here so Windows
+                    # \ directory separators don't come into play.
+                    dirname = os.path.dirname(include)
+                    # basename is used to avoid issues with absolute
+                    # paths (which on Windows can include the drive).
+                    while os.path.basename(dirname):
+                        subdirs.add(dirname)
+                        dirname = os.path.dirname(dirname)
+
+                matcher = matchmod.match(repo.root, '', [],
+                                         include=includes, exclude=excludes,
+                                         default='relpath')
+                if subdirs:
+                    matcher = matchmod.forceincludematcher(matcher, subdirs)
+                matchers.append(matcher)
+        except IOError:
+            pass
+
+    if not matchers:
+        result = matchmod.always(repo.root, '')
+    elif len(matchers) == 1:
+        result = matchers[0]
+    else:
+        result = matchmod.unionmatcher(matchers)
+
+    if includetemp:
+        tempincludes = readtemporaryincludes(repo)
+        result = matchmod.forceincludematcher(result, tempincludes)
+
+    repo._sparsematchercache[key] = result
+
+    return result