192 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
192 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
193 self.retry = ui.configint('lfs', 'retry') |
193 self.retry = ui.configint('lfs', 'retry') |
194 |
194 |
195 def writebatch(self, pointers, fromstore): |
195 def writebatch(self, pointers, fromstore): |
196 """Batch upload from local to remote blobstore.""" |
196 """Batch upload from local to remote blobstore.""" |
197 self._batch(pointers, fromstore, 'upload') |
197 self._batch(_deduplicate(pointers), fromstore, 'upload') |
198 |
198 |
199 def readbatch(self, pointers, tostore): |
199 def readbatch(self, pointers, tostore): |
200 """Batch download from remote to local blostore.""" |
200 """Batch download from remote to local blostore.""" |
201 self._batch(pointers, tostore, 'download') |
201 self._batch(_deduplicate(pointers), tostore, 'download') |
202 |
202 |
203 def _batchrequest(self, pointers, action): |
203 def _batchrequest(self, pointers, action): |
204 """Get metadata about objects pointed by pointers for given action |
204 """Get metadata about objects pointed by pointers for given action |
205 |
205 |
206 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
206 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
397 def __init__(self, repo, url): |
397 def __init__(self, repo, url): |
398 fullpath = repo.vfs.join('lfs', url.path) |
398 fullpath = repo.vfs.join('lfs', url.path) |
399 self.vfs = lfsvfs(fullpath) |
399 self.vfs = lfsvfs(fullpath) |
400 |
400 |
401 def writebatch(self, pointers, fromstore): |
401 def writebatch(self, pointers, fromstore): |
402 for p in pointers: |
402 for p in _deduplicate(pointers): |
403 content = fromstore.read(p.oid(), verify=True) |
403 content = fromstore.read(p.oid(), verify=True) |
404 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: |
404 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: |
405 fp.write(content) |
405 fp.write(content) |
406 |
406 |
407 def readbatch(self, pointers, tostore): |
407 def readbatch(self, pointers, tostore): |
408 for p in pointers: |
408 for p in _deduplicate(pointers): |
409 with self.vfs(p.oid(), 'rb') as fp: |
409 with self.vfs(p.oid(), 'rb') as fp: |
410 tostore.download(p.oid(), fp) |
410 tostore.download(p.oid(), fp) |
411 |
411 |
412 class _nullremote(object): |
412 class _nullremote(object): |
413 """Null store storing blobs to /dev/null.""" |
413 """Null store storing blobs to /dev/null.""" |
441 'http': _gitlfsremote, |
441 'http': _gitlfsremote, |
442 'file': _dummyremote, |
442 'file': _dummyremote, |
443 'null': _nullremote, |
443 'null': _nullremote, |
444 None: _promptremote, |
444 None: _promptremote, |
445 } |
445 } |
|
446 |
|
447 def _deduplicate(pointers): |
|
448 """Remove any duplicate oids that exist in the list""" |
|
449 reduced = util.sortdict() |
|
450 for p in pointers: |
|
451 reduced[p.oid()] = p |
|
452 return reduced.values() |
446 |
453 |
447 def _verify(oid, content): |
454 def _verify(oid, content): |
448 realoid = hashlib.sha256(content).hexdigest() |
455 realoid = hashlib.sha256(content).hexdigest() |
449 if realoid != oid: |
456 if realoid != oid: |
450 raise error.Abort(_('detected corrupt lfs object: %s') % oid, |
457 raise error.Abort(_('detected corrupt lfs object: %s') % oid, |