largefiles: stat all largefiles in one batch before downloading
This avoids a lot of expensive roundtrips to remote repositories ... but might
be slightly slower for local operations.
This will also change some aborts on missing files to warnings. That will in
some situations make it possible to continue working on a repository with
missing largefiles.
--- a/hgext/largefiles/basestore.py Mon Apr 15 23:34:36 2013 +0200
+++ b/hgext/largefiles/basestore.py Mon Apr 15 23:37:43 2013 +0200
@@ -62,12 +62,19 @@
util.makedirs(lfutil.storepath(self.repo, ''))
at = 0
+ available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
+ if not available.get(hash):
+ ui.warn(_('%s: largefile %s not available from %s\n')
+ % (filename, hash, self.url))
+ missing.append(filename)
+ continue
+
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(storefilename + '.tmp',
createmode=self.repo.store.createmode)
--- a/hgext/largefiles/remotestore.py Mon Apr 15 23:34:36 2013 +0200
+++ b/hgext/largefiles/remotestore.py Mon Apr 15 23:37:43 2013 +0200
@@ -47,16 +47,6 @@
fd.close()
def _getfile(self, tmpfile, filename, hash):
- # quit if the largefile isn't there
- stat = self._stat([hash])[hash]
- if stat == 1:
- raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
- elif stat == 2:
- raise util.Abort(_('remotestore: largefile %s is missing') % hash)
- elif stat != 0:
- raise RuntimeError('error getting file: unexpected response from '
- 'statlfile (%r)' % stat)
-
try:
chunks = self._get(hash)
except urllib2.HTTPError, e:
--- a/hgext/largefiles/wirestore.py Mon Apr 15 23:34:36 2013 +0200
+++ b/hgext/largefiles/wirestore.py Mon Apr 15 23:37:43 2013 +0200
@@ -26,8 +26,9 @@
return self.remote.getlfile(hash)
def _stat(self, hashes):
- '''For each hash, return 2 if the largefile is missing, 1 if it has a
- mismatched checksum, or 0 if it is in good condition'''
+ '''For each hash, return 0 if it is available, other values if not.
+ It is usually 2 if the largefile is missing, but might be 1 the server
+ has a corrupted copy.'''
batch = self.remote.batch()
futures = {}
for hash in hashes:
--- a/tests/test-largefiles-cache.t Mon Apr 15 23:34:36 2013 +0200
+++ b/tests/test-largefiles-cache.t Mon Apr 15 23:37:43 2013 +0200
@@ -47,7 +47,7 @@
$ hg update -r0
getting changed largefiles
- error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+ large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
0 largefiles updated, 0 removed
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg status
@@ -64,7 +64,7 @@
$ hg update -r0
getting changed largefiles
- error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+ large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
0 largefiles updated, 0 removed
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg status
--- a/tests/test-largefiles.t Mon Apr 15 23:34:36 2013 +0200
+++ b/tests/test-largefiles.t Mon Apr 15 23:37:43 2013 +0200
@@ -1306,7 +1306,7 @@
$ rm ${USERCACHE}/7838695e10da2bb75ac1156565f40a2595fa2fa0
$ hg up -r 6
getting changed largefiles
- error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+ large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
1 largefiles updated, 2 removed
4 files updated, 0 files merged, 2 files removed, 0 files unresolved
$ rm normal3
@@ -1327,7 +1327,7 @@
! normal3
$ hg up -Cr.
getting changed largefiles
- error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+ large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
0 largefiles updated, 0 removed
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg st
@@ -1349,7 +1349,7 @@
4 files updated, 0 files merged, 0 files removed, 0 files unresolved
(branch merge, don't forget to commit)
getting changed largefiles
- error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+ large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
1 largefiles updated, 0 removed
$ hg rollback -q
@@ -1773,8 +1773,11 @@
$ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 .
$ hg -R http-clone up --config largefiles.usercache=http-clone-usercache
getting changed largefiles
- abort: remotestore: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 is missing
- [255]
+ f1: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 not available from http://localhost:$HGPORT2/
+ 0 largefiles updated, 0 removed
+ 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ hg -R http-clone st
+ ! f1
$ hg -R http-clone up -Cqr null
largefiles pulled on update - a largefile corrupted on the server:
@@ -1811,9 +1814,9 @@
getting changed largefiles
using http://localhost:$HGPORT2/
sending capabilities command
+ sending batch command
getting largefiles: 0/1 lfile (0.00%)
getting f1:02a439e5c31c526465ab1a0ca1f431f76b827b90
- sending batch command
sending getlfile command
found 02a439e5c31c526465ab1a0ca1f431f76b827b90 in store
1 largefiles updated, 0 removed
--- a/tests/test-lfconvert.t Mon Apr 15 23:34:36 2013 +0200
+++ b/tests/test-lfconvert.t Mon Apr 15 23:37:43 2013 +0200
@@ -343,7 +343,7 @@
$ rm largefiles-repo/.hg/largefiles/*
$ hg lfconvert --to-normal issue3519 normalized3519
initializing destination normalized3519
- error getting id 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 from url file:$TESTTMP/largefiles-repo for file large: can't get file locally (glob)
+ large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:$TESTTMP/largefiles-repo
abort: missing largefile 'large' from revision d4892ec57ce212905215fad1d9018f56b99202ad
[255]