changeset 37762:7269b87f817c

scmutil: teach the file prefetch hook to handle multiple commits The remainder of the commands that need prefetch deal with multiple revisions. I initially coded this as a separate hook, but then it needed a list of files to handle `diff` and `grep`, so it didn't seem worth keeping them separate. Not every matcher will emit bad file messages (some are built from a list of files that are known to exist). But it seems better to filter this in one place than to push this on either each caller or each hook implementation.
author Matt Harbison <matt_harbison@yahoo.com>
date Sat, 14 Apr 2018 18:50:45 -0400
parents ff6b0a20849d
children b54404d66f7e
files hgext/lfs/wrapper.py mercurial/archival.py mercurial/cmdutil.py mercurial/merge.py mercurial/scmutil.py mercurial/subrepo.py tests/test-lfs-test-server.t
diffstat 7 files changed, 45 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/hgext/lfs/wrapper.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/hgext/lfs/wrapper.py	Sat Apr 14 18:50:45 2018 -0400
@@ -244,17 +244,21 @@
     if 'lfs' in destrepo.requirements:
         destrepo.vfs.append('hgrc', util.tonativeeol('\n[extensions]\nlfs=\n'))
 
-def _prefetchfiles(repo, ctx, files):
+def _prefetchfiles(repo, revs, match):
     """Ensure that required LFS blobs are present, fetching them as a group if
     needed."""
     pointers = []
+    oids = set()
     localstore = repo.svfs.lfslocalblobstore
 
-    for f in files:
-        p = pointerfromctx(ctx, f)
-        if p and not localstore.has(p.oid()):
-            p.filename = f
-            pointers.append(p)
+    for rev in revs:
+        ctx = repo[rev]
+        for f in ctx.walk(match):
+            p = pointerfromctx(ctx, f)
+            if p and p.oid() not in oids and not localstore.has(p.oid()):
+                p.filename = f
+                pointers.append(p)
+                oids.add(p.oid())
 
     if pointers:
         # Recalculating the repo store here allows 'paths.default' that is set
--- a/mercurial/archival.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/mercurial/archival.py	Sat Apr 14 18:50:45 2018 -0400
@@ -320,7 +320,8 @@
     total = len(files)
     if total:
         files.sort()
-        scmutil.fileprefetchhooks(repo, ctx, files)
+        scmutil.prefetchfiles(repo, [ctx.rev()],
+                              scmutil.matchfiles(repo, files))
         repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
         for i, f in enumerate(files):
             ff = ctx.flags(f)
--- a/mercurial/cmdutil.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/mercurial/cmdutil.py	Sat Apr 14 18:50:45 2018 -0400
@@ -2292,16 +2292,15 @@
         mfnode = ctx.manifestnode()
         try:
             if mfnode and mfl[mfnode].find(file)[0]:
-                scmutil.fileprefetchhooks(repo, ctx, [file])
+                scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
                 write(file)
                 return 0
         except KeyError:
             pass
 
-    files = [f for f in ctx.walk(matcher)]
-    scmutil.fileprefetchhooks(repo, ctx, files)
-
-    for abs in files:
+    scmutil.prefetchfiles(repo, [ctx.rev()], matcher)
+
+    for abs in ctx.walk(matcher):
         write(abs)
         err = 0
 
@@ -2979,8 +2978,11 @@
                 _revertprefetch(repo, ctx,
                                 *[actions[name][0] for name in needdata])
             oplist = [actions[name][0] for name in needdata]
-            prefetch = scmutil.fileprefetchhooks
-            prefetch(repo, ctx, [f for sublist in oplist for f in sublist])
+            prefetch = scmutil.prefetchfiles
+            matchfiles = scmutil.matchfiles
+            prefetch(repo, [ctx.rev()],
+                     matchfiles(repo,
+                                [f for sublist in oplist for f in sublist]))
             _performrevert(repo, parents, ctx, actions, interactive, tobackup)
 
         if targetsubs:
--- a/mercurial/merge.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/mercurial/merge.py	Sat Apr 14 18:50:45 2018 -0400
@@ -1465,7 +1465,7 @@
         yield i, f
 
 def _prefetchfiles(repo, ctx, actions):
-    """Invoke ``scmutil.fileprefetchhooks()`` for the files relevant to the dict
+    """Invoke ``scmutil.prefetchfiles()`` for the files relevant to the dict
     of merge actions.  ``ctx`` is the context being merged in."""
 
     # Skipping 'a', 'am', 'f', 'r', 'dm', 'e', 'k', 'p' and 'pr', because they
@@ -1473,8 +1473,11 @@
     # changed/deleted never resolves to something from the remote side.
     oplist = [actions[a] for a in (ACTION_GET, ACTION_DELETED_CHANGED,
                                    ACTION_LOCAL_DIR_RENAME_GET, ACTION_MERGE)]
-    prefetch = scmutil.fileprefetchhooks
-    prefetch(repo, ctx, [f for sublist in oplist for f, args, msg in sublist])
+    prefetch = scmutil.prefetchfiles
+    matchfiles = scmutil.matchfiles
+    prefetch(repo, [ctx.rev()],
+             matchfiles(repo,
+                        [f for sublist in oplist for f, args, msg in sublist]))
 
 @attr.s(frozen=True)
 class updateresult(object):
--- a/mercurial/scmutil.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/mercurial/scmutil.py	Sat Apr 14 18:50:45 2018 -0400
@@ -1357,9 +1357,20 @@
     'unbundle',
 ]
 
-# a list of (repo, ctx, files) functions called by various commands to allow
-# extensions to ensure the corresponding files are available locally, before the
-# command uses them.
+def prefetchfiles(repo, revs, match):
+    """Invokes the registered file prefetch functions, allowing extensions to
+    ensure the corresponding files are available locally, before the command
+    uses them."""
+    if match:
+        # The command itself will complain about files that don't exist, so
+        # don't duplicate the message.
+        match = matchmod.badmatch(match, lambda fn, msg: None)
+    else:
+        match = matchall(repo)
+
+    fileprefetchhooks(repo, revs, match)
+
+# a list of (repo, revs, match) prefetch functions
 fileprefetchhooks = util.hooks()
 
 # A marker that tells the evolve extension to suppress its own reporting
--- a/mercurial/subrepo.py	Mon Apr 16 23:39:30 2018 -0400
+++ b/mercurial/subrepo.py	Sat Apr 14 18:50:45 2018 -0400
@@ -562,7 +562,8 @@
             files = [f for f in files if match(f)]
         rev = self._state[1]
         ctx = self._repo[rev]
-        scmutil.fileprefetchhooks(self._repo, ctx, files)
+        scmutil.prefetchfiles(self._repo, [ctx.rev()],
+                              scmutil.matchfiles(self._repo, files))
         total = abstractsubrepo.archive(self, archiver, prefix, match)
         for subpath in ctx.substate:
             s = subrepo(ctx, subpath, True)
--- a/tests/test-lfs-test-server.t	Mon Apr 16 23:39:30 2018 -0400
+++ b/tests/test-lfs-test-server.t	Sat Apr 14 18:50:45 2018 -0400
@@ -616,7 +616,7 @@
 Cat will prefetch blobs in a group
 
   $ rm -rf .hg/store/lfs `hg config lfs.usercache`
-  $ hg cat --debug -r 1 a b c
+  $ hg cat --debug -r 1 a b c nonexistent
   http auth: user foo, password ***
   http auth: user foo, password ***
   Status: 200
@@ -681,6 +681,7 @@
   THIS-IS-LFS
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
   ANOTHER-LARGE-FILE
+  nonexistent: no such file in rev dfca2c9e2ef2
 
 Revert will prefetch blobs in a group