hgext/lfs/blobstore.py
changeset 35927 9b413478f261
parent 35881 fa993c3c8462
child 36455 9e3cb58c7ab3
equal deleted inserted replaced
35926:264b90a060b7 35927:9b413478f261
   192         self.urlopener = urlmod.opener(ui, authinfo, useragent)
   192         self.urlopener = urlmod.opener(ui, authinfo, useragent)
   193         self.retry = ui.configint('lfs', 'retry')
   193         self.retry = ui.configint('lfs', 'retry')
   194 
   194 
   195     def writebatch(self, pointers, fromstore):
   195     def writebatch(self, pointers, fromstore):
   196         """Batch upload from local to remote blobstore."""
   196         """Batch upload from local to remote blobstore."""
   197         self._batch(pointers, fromstore, 'upload')
   197         self._batch(_deduplicate(pointers), fromstore, 'upload')
   198 
   198 
   199     def readbatch(self, pointers, tostore):
   199     def readbatch(self, pointers, tostore):
   200         """Batch download from remote to local blostore."""
   200         """Batch download from remote to local blostore."""
   201         self._batch(pointers, tostore, 'download')
   201         self._batch(_deduplicate(pointers), tostore, 'download')
   202 
   202 
   203     def _batchrequest(self, pointers, action):
   203     def _batchrequest(self, pointers, action):
   204         """Get metadata about objects pointed by pointers for given action
   204         """Get metadata about objects pointed by pointers for given action
   205 
   205 
   206         Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
   206         Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
   397     def __init__(self, repo, url):
   397     def __init__(self, repo, url):
   398         fullpath = repo.vfs.join('lfs', url.path)
   398         fullpath = repo.vfs.join('lfs', url.path)
   399         self.vfs = lfsvfs(fullpath)
   399         self.vfs = lfsvfs(fullpath)
   400 
   400 
   401     def writebatch(self, pointers, fromstore):
   401     def writebatch(self, pointers, fromstore):
   402         for p in pointers:
   402         for p in _deduplicate(pointers):
   403             content = fromstore.read(p.oid(), verify=True)
   403             content = fromstore.read(p.oid(), verify=True)
   404             with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
   404             with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
   405                 fp.write(content)
   405                 fp.write(content)
   406 
   406 
   407     def readbatch(self, pointers, tostore):
   407     def readbatch(self, pointers, tostore):
   408         for p in pointers:
   408         for p in _deduplicate(pointers):
   409             with self.vfs(p.oid(), 'rb') as fp:
   409             with self.vfs(p.oid(), 'rb') as fp:
   410                 tostore.download(p.oid(), fp)
   410                 tostore.download(p.oid(), fp)
   411 
   411 
   412 class _nullremote(object):
   412 class _nullremote(object):
   413     """Null store storing blobs to /dev/null."""
   413     """Null store storing blobs to /dev/null."""
   441     'http': _gitlfsremote,
   441     'http': _gitlfsremote,
   442     'file': _dummyremote,
   442     'file': _dummyremote,
   443     'null': _nullremote,
   443     'null': _nullremote,
   444     None: _promptremote,
   444     None: _promptremote,
   445 }
   445 }
       
   446 
       
   447 def _deduplicate(pointers):
       
   448     """Remove any duplicate oids that exist in the list"""
       
   449     reduced = util.sortdict()
       
   450     for p in pointers:
       
   451         reduced[p.oid()] = p
       
   452     return reduced.values()
   446 
   453 
   447 def _verify(oid, content):
   454 def _verify(oid, content):
   448     realoid = hashlib.sha256(content).hexdigest()
   455     realoid = hashlib.sha256(content).hexdigest()
   449     if realoid != oid:
   456     if realoid != oid:
   450         raise error.Abort(_('detected corrupt lfs object: %s') % oid,
   457         raise error.Abort(_('detected corrupt lfs object: %s') % oid,