largefiles: change basestore._verifyfile to take list of files to check
Makes it easier to use batch stat calls in remotestore to decrease
number of round trips.
--- a/hgext/largefiles/basestore.py Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/basestore.py Tue May 03 23:31:32 2016 +0200
@@ -116,19 +116,26 @@
'''Verify the existence (and, optionally, contents) of every big
file revision referenced by every changeset in revs.
Return 0 if all is well, non-zero on any errors.'''
- failed = False
self.ui.status(_('searching %d changesets for largefiles\n') %
len(revs))
verified = set() # set of (filename, filenode) tuples
-
+ filestocheck = [] # list of (cset, filename, expectedhash)
for rev in revs:
cctx = self.repo[rev]
cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
for standin in cctx:
- if self._verifyfile(cctx, cset, contents, standin, verified):
- failed = True
+ filename = lfutil.splitstandin(standin)
+ if filename:
+ fctx = cctx[standin]
+ key = (filename, fctx.filenode())
+ if key not in verified:
+ verified.add(key)
+ expectedhash = fctx.data()[0:40]
+ filestocheck.append((cset, filename, expectedhash))
+
+ failed = self._verifyfiles(contents, filestocheck)
numrevs = len(verified)
numlfiles = len(set([fname for (fname, fnode) in verified]))
@@ -150,13 +157,11 @@
exist in the store).'''
raise NotImplementedError('abstract method')
- def _verifyfile(self, cctx, cset, contents, standin, verified):
- '''Perform the actual verification of a file in the store.
- 'cset' is only used in warnings.
+ def _verifyfiles(self, contents, filestocheck):
+ '''Perform the actual verification of files in the store.
'contents' controls verification of content hash.
- 'standin' is the standin path of the largefile to verify.
- 'verified' is maintained as a set of already verified files.
- Returns _true_ if it is a standin and any problems are found!
+ 'filestocheck' is list of files to check.
+ Returns _true_ if any problems are found!
'''
raise NotImplementedError('abstract method')
--- a/hgext/largefiles/localstore.py Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/localstore.py Tue May 03 23:31:32 2016 +0200
@@ -42,29 +42,20 @@
with open(path, 'rb') as fd:
return lfutil.copyandhash(fd, tmpfile)
- def _verifyfile(self, cctx, cset, contents, standin, verified):
- filename = lfutil.splitstandin(standin)
- if not filename:
- return False
- fctx = cctx[standin]
- key = (filename, fctx.filenode())
- if key in verified:
- return False
-
- expecthash = fctx.data()[0:40]
- storepath, exists = lfutil.findstorepath(self.remote, expecthash)
- verified.add(key)
- if not exists:
- self.ui.warn(
- _('changeset %s: %s references missing %s\n')
- % (cset, filename, storepath))
- return True # failed
-
- if contents:
- actualhash = lfutil.hashfile(storepath)
- if actualhash != expecthash:
+ def _verifyfiles(self, contents, filestocheck):
+ failed = False
+ for cset, filename, expectedhash in filestocheck:
+ storepath, exists = lfutil.findstorepath(self.remote, expectedhash)
+ if not exists:
self.ui.warn(
- _('changeset %s: %s references corrupted %s\n')
+ _('changeset %s: %s references missing %s\n')
% (cset, filename, storepath))
- return True # failed
- return False
+ failed = True
+ elif contents:
+ actualhash = lfutil.hashfile(storepath)
+ if actualhash != expectedhash:
+ self.ui.warn(
+ _('changeset %s: %s references corrupted %s\n')
+ % (cset, filename, storepath))
+ failed = True
+ return failed
--- a/hgext/largefiles/remotestore.py Mon Apr 11 21:33:07 2016 +0000
+++ b/hgext/largefiles/remotestore.py Tue May 03 23:31:32 2016 +0200
@@ -65,34 +65,25 @@
return lfutil.copyandhash(chunks, tmpfile)
- def _verifyfile(self, cctx, cset, contents, standin, verified):
- filename = lfutil.splitstandin(standin)
- if not filename:
- return False
- fctx = cctx[standin]
- key = (filename, fctx.filenode())
- if key in verified:
- return False
-
- verified.add(key)
-
- expecthash = fctx.data()[0:40]
- stat = self._stat([expecthash])[expecthash]
- if not stat:
- return False
- elif stat == 1:
- self.ui.warn(
- _('changeset %s: %s: contents differ\n')
- % (cset, filename))
- return True # failed
- elif stat == 2:
- self.ui.warn(
- _('changeset %s: %s missing\n')
- % (cset, filename))
- return True # failed
- else:
- raise RuntimeError('verify failed: unexpected response from '
- 'statlfile (%r)' % stat)
+ def _verifyfiles(self, contents, filestocheck):
+ failed = False
+ for cset, filename, expectedhash in filestocheck:
+ stat = self._stat([expectedhash])[expectedhash]
+ if stat:
+ if stat == 1:
+ self.ui.warn(
+ _('changeset %s: %s: contents differ\n')
+ % (cset, filename))
+ failed = True
+ elif stat == 2:
+ self.ui.warn(
+ _('changeset %s: %s missing\n')
+ % (cset, filename))
+ failed = True
+ else:
+ raise RuntimeError('verify failed: unexpected response '
+ 'from statlfile (%r)' % stat)
+ return failed
def batch(self):
'''Support for remote batching.'''