Mercurial > hg
diff hgext/largefiles/basestore.py @ 15168:cfccd3bee7b3
hgext: add largefiles extension
This code has a number of contributors and a complicated history prior to its
introduction that can be seen by visiting:
https://developers.kilnhg.com/Repo/Kiln/largefiles/largefiles
http://hg.gerg.ca/hg-bfiles
and looking at the included copyright notices and contributors list.
author | various |
---|---|
date | Sat, 24 Sep 2011 17:35:45 +0200 |
parents | |
children | aa262fff87ac |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hgext/largefiles/basestore.py Sat Sep 24 17:35:45 2011 +0200 @@ -0,0 +1,201 @@ +# Copyright 2009-2010 Gregory P. Ward +# Copyright 2009-2010 Intelerad Medical Systems Incorporated +# Copyright 2010-2011 Fog Creek Software +# Copyright 2010-2011 Unity Technologies +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +'''Base class for store implementations and store-related utility code.''' + +import os +import tempfile +import binascii +import re + +from mercurial import util, node, hg +from mercurial.i18n import _ + +import lfutil + +class StoreError(Exception): + '''Raised when there is a problem getting files from or putting + files to a central store.''' + def __init__(self, filename, hash, url, detail): + self.filename = filename + self.hash = hash + self.url = url + self.detail = detail + + def longmessage(self): + if self.url: + return ('%s: %s\n' + '(failed URL: %s)\n' + % (self.filename, self.detail, self.url)) + else: + return ('%s: %s\n' + '(no default or default-push path set in hgrc)\n' + % (self.filename, self.detail)) + + def __str__(self): + return "%s: %s" % (self.url, self.detail) + +class basestore(object): + def __init__(self, ui, repo, url): + self.ui = ui + self.repo = repo + self.url = url + + def put(self, source, hash): + '''Put source file into the store under <filename>/<hash>.''' + raise NotImplementedError('abstract method') + + def exists(self, hash): + '''Check to see if the store contains the given hash.''' + raise NotImplementedError('abstract method') + + def get(self, files): + '''Get the specified largefiles from the store and write to local + files under repo.root. files is a list of (filename, hash) + tuples. Return (success, missing), lists of files successfuly + downloaded and those not found in the store. success is a list + of (filename, hash) tuples; missing is a list of filenames that + we could not get. (The detailed error message will already have + been presented to the user, so missing is just supplied as a + summary.)''' + success = [] + missing = [] + ui = self.ui + + at = 0 + for filename, hash in files: + ui.progress(_('getting largefiles'), at, unit='lfile', + total=len(files)) + at += 1 + ui.note(_('getting %s:%s\n') % (filename, hash)) + + cachefilename = lfutil.cachepath(self.repo, hash) + cachedir = os.path.dirname(cachefilename) + + # No need to pass mode='wb' to fdopen(), since mkstemp() already + # opened the file in binary mode. + (tmpfd, tmpfilename) = tempfile.mkstemp( + dir=cachedir, prefix=os.path.basename(filename)) + tmpfile = os.fdopen(tmpfd, 'w') + + try: + hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash)) + except StoreError, err: + ui.warn(err.longmessage()) + hhash = "" + + if hhash != hash: + if hhash != "": + ui.warn(_('%s: data corruption (expected %s, got %s)\n') + % (filename, hash, hhash)) + tmpfile.close() # no-op if it's already closed + os.remove(tmpfilename) + missing.append(filename) + continue + + if os.path.exists(cachefilename): # Windows + os.remove(cachefilename) + os.rename(tmpfilename, cachefilename) + lfutil.linktosystemcache(self.repo, hash) + success.append((filename, hhash)) + + ui.progress(_('getting largefiles'), None) + return (success, missing) + + def verify(self, revs, contents=False): + '''Verify the existence (and, optionally, contents) of every big + file revision referenced by every changeset in revs. + Return 0 if all is well, non-zero on any errors.''' + write = self.ui.write + failed = False + + write(_('searching %d changesets for largefiles\n') % len(revs)) + verified = set() # set of (filename, filenode) tuples + + for rev in revs: + cctx = self.repo[rev] + cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) + + failed = lfutil.any_(self._verifyfile( + cctx, cset, contents, standin, verified) for standin in cctx) + + num_revs = len(verified) + num_lfiles = len(set([fname for (fname, fnode) in verified])) + if contents: + write(_('verified contents of %d revisions of %d largefiles\n') + % (num_revs, num_lfiles)) + else: + write(_('verified existence of %d revisions of %d largefiles\n') + % (num_revs, num_lfiles)) + + return int(failed) + + def _getfile(self, tmpfile, filename, hash): + '''Fetch one revision of one file from the store and write it + to tmpfile. Compute the hash of the file on-the-fly as it + downloads and return the binary hash. Close tmpfile. Raise + StoreError if unable to download the file (e.g. it does not + exist in the store).''' + raise NotImplementedError('abstract method') + + def _verifyfile(self, cctx, cset, contents, standin, verified): + '''Perform the actual verification of a file in the store. + ''' + raise NotImplementedError('abstract method') + +import localstore, wirestore + +_storeprovider = { + 'file': [localstore.localstore], + 'http': [wirestore.wirestore], + 'https': [wirestore.wirestore], + 'ssh': [wirestore.wirestore], + } + +_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') + +# During clone this function is passed the src's ui object +# but it needs the dest's ui object so it can read out of +# the config file. Use repo.ui instead. +def _openstore(repo, remote=None, put=False): + ui = repo.ui + + if not remote: + path = getattr(repo, 'lfpullsource', None) or \ + ui.expandpath('default-push', 'default') + # If 'default-push' and 'default' can't be expanded + # they are just returned. In that case use the empty string which + # use the filescheme. + if path == 'default-push' or path == 'default': + path = '' + remote = repo + else: + remote = hg.peer(repo, {}, path) + + # The path could be a scheme so use Mercurial's normal functionality + # to resolve the scheme to a repository and use its path + path = hasattr(remote, 'url') and remote.url() or remote.path + + match = _scheme_re.match(path) + if not match: # regular filesystem path + scheme = 'file' + else: + scheme = match.group(1) + + try: + storeproviders = _storeprovider[scheme] + except KeyError: + raise util.Abort(_('unsupported URL scheme %r') % scheme) + + for class_obj in storeproviders: + try: + return class_obj(ui, repo, remote) + except lfutil.storeprotonotcapable: + pass + + raise util.Abort(_('%s does not appear to be a lfile store'), path)