changeset 29067:207c0db08953

largefiles: change basestore._verifyfile to take list of files to check

Makes it easier to use batch stat calls in remotestore to decrease
number of round trips.

--- a/hgext/largefiles/basestore.py	Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/basestore.py	Tue May 03 23:31:32 2016 +0200
@@ -116,19 +116,26 @@
         '''Verify the existence (and, optionally, contents) of every big
         file revision referenced by every changeset in revs.
         Return 0 if all is well, non-zero on any errors.'''
-        failed = False
 
         self.ui.status(_('searching %d changesets for largefiles\n') %
                        len(revs))
         verified = set()                # set of (filename, filenode) tuples
-
+        filestocheck = []               # list of (cset, filename, expectedhash)
         for rev in revs:
             cctx = self.repo[rev]
             cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
 
             for standin in cctx:
-                if self._verifyfile(cctx, cset, contents, standin, verified):
-                    failed = True
+                filename = lfutil.splitstandin(standin)
+                if filename:
+                    fctx = cctx[standin]
+                    key = (filename, fctx.filenode())
+                    if key not in verified:
+                        verified.add(key)
+                        expectedhash = fctx.data()[0:40]
+                        filestocheck.append((cset, filename, expectedhash))
+
+        failed = self._verifyfiles(contents, filestocheck)
 
         numrevs = len(verified)
         numlfiles = len(set([fname for (fname, fnode) in verified]))
@@ -150,13 +157,11 @@
         exist in the store).'''
         raise NotImplementedError('abstract method')
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        '''Perform the actual verification of a file in the store.
-        'cset' is only used in warnings.
+    def _verifyfiles(self, contents, filestocheck):
+        '''Perform the actual verification of files in the store.
         'contents' controls verification of content hash.
-        'standin' is the standin path of the largefile to verify.
-        'verified' is maintained as a set of already verified files.
-        Returns _true_ if it is a standin and any problems are found!
+        'filestocheck' is list of files to check.
+        Returns _true_ if any problems are found!
         '''
         raise NotImplementedError('abstract method')
 

--- a/hgext/largefiles/localstore.py	Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/localstore.py	Tue May 03 23:31:32 2016 +0200
@@ -42,29 +42,20 @@
         with open(path, 'rb') as fd:
             return lfutil.copyandhash(fd, tmpfile)
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        filename = lfutil.splitstandin(standin)
-        if not filename:
-            return False
-        fctx = cctx[standin]
-        key = (filename, fctx.filenode())
-        if key in verified:
-            return False
-
-        expecthash = fctx.data()[0:40]
-        storepath, exists = lfutil.findstorepath(self.remote, expecthash)
-        verified.add(key)
-        if not exists:
-            self.ui.warn(
-                _('changeset %s: %s references missing %s\n')
-                % (cset, filename, storepath))
-            return True                 # failed
-
-        if contents:
-            actualhash = lfutil.hashfile(storepath)
-            if actualhash != expecthash:
+    def _verifyfiles(self, contents, filestocheck):
+        failed = False
+        for cset, filename, expectedhash in filestocheck:
+            storepath, exists = lfutil.findstorepath(self.remote, expectedhash)
+            if not exists:
                 self.ui.warn(
-                    _('changeset %s: %s references corrupted %s\n')
+                    _('changeset %s: %s references missing %s\n')
                     % (cset, filename, storepath))
-                return True             # failed
-        return False
+                failed = True
+            elif contents:
+                actualhash = lfutil.hashfile(storepath)
+                if actualhash != expectedhash:
+                    self.ui.warn(
+                        _('changeset %s: %s references corrupted %s\n')
+                        % (cset, filename, storepath))
+                    failed = True
+        return failed

--- a/hgext/largefiles/remotestore.py	Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/remotestore.py	Tue May 03 23:31:32 2016 +0200
@@ -65,34 +65,25 @@
 
         return lfutil.copyandhash(chunks, tmpfile)
 
-    def _verifyfile(self, cctx, cset, contents, standin, verified):
-        filename = lfutil.splitstandin(standin)
-        if not filename:
-            return False
-        fctx = cctx[standin]
-        key = (filename, fctx.filenode())
-        if key in verified:
-            return False
-
-        verified.add(key)
-
-        expecthash = fctx.data()[0:40]
-        stat = self._stat([expecthash])[expecthash]
-        if not stat:
-            return False
-        elif stat == 1:
-            self.ui.warn(
-                _('changeset %s: %s: contents differ\n')
-                % (cset, filename))
-            return True # failed
-        elif stat == 2:
-            self.ui.warn(
-                _('changeset %s: %s missing\n')
-                % (cset, filename))
-            return True # failed
-        else:
-            raise RuntimeError('verify failed: unexpected response from '
-                               'statlfile (%r)' % stat)
+    def _verifyfiles(self, contents, filestocheck):
+        failed = False
+        for cset, filename, expectedhash in filestocheck:
+            stat = self._stat([expectedhash])[expectedhash]
+            if stat:
+                if stat == 1:
+                    self.ui.warn(
+                        _('changeset %s: %s: contents differ\n')
+                        % (cset, filename))
+                    failed = True
+                elif stat == 2:
+                    self.ui.warn(
+                        _('changeset %s: %s missing\n')
+                        % (cset, filename))
+                    failed = True
+                else:
+                    raise RuntimeError('verify failed: unexpected response '
+                                       'from statlfile (%r)' % stat)
+        return failed
 
     def batch(self):
         '''Support for remote batching.'''