Mercurial > hg
changeset 6840:80e51429cb9a
introduce store classes
move store walking from streamclone.py into store.py
| author | Adrian Buehlmann <adrian@cadifra.com> |
|---|---|
| date | Thu, 24 Jul 2008 16:32:52 +0200 |
| parents | 01db3e101362 |
| children | b114a8c7998f |
| files | mercurial/localrepo.py mercurial/statichttprepo.py mercurial/store.py mercurial/streamclone.py |
| diffstat | 4 files changed, 126 insertions(+), 85 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/localrepo.py Thu Jul 24 16:32:51 2008 +0200 +++ b/mercurial/localrepo.py Thu Jul 24 16:32:52 2008 +0200 @@ -60,30 +60,13 @@ if r not in self.supported: raise repo.RepoError(_("requirement '%s' not supported") % r) - # setup store - if "store" in requirements: - self.encodefn = store.encodefilename - self.decodefn = store.decodefilename - self.spath = os.path.join(self.path, "store") - else: - self.encodefn = lambda x: x - self.decodefn = lambda x: x - self.spath = self.path + self.store = store.store(requirements, self.path) - try: - # files in .hg/ will be created using this mode - mode = os.stat(self.spath).st_mode - # avoid some useless chmods - if (0777 & ~util._umask) == (0777 & mode): - mode = None - except OSError: - mode = None - - self._createmode = mode - self.opener.createmode = mode - sopener = util.opener(self.spath) - sopener.createmode = mode - self.sopener = store.encodedopener(sopener, self.encodefn) + self.spath = self.store.path + self.sopener = self.store.opener + self.sjoin = self.store.join + self._createmode = self.store.createmode + self.opener.createmode = self.store.createmode self.ui = ui.ui(parentui=parentui) try: @@ -481,10 +464,6 @@ def join(self, f): return os.path.join(self.path, f) - def sjoin(self, f): - f = self.encodefn(f) - return os.path.join(self.spath, f) - def wjoin(self, f): return os.path.join(self.root, f) @@ -2061,6 +2040,25 @@ return self.stream_in(remote) return self.pull(remote, heads) + def storefiles(self): + '''get all *.i and *.d files in the store + + Returns (list of (filename, size), total_bytes)''' + + lock = None + try: + self.ui.debug('scanning\n') + entries = [] + total_bytes = 0 + # get consistent snapshot of repo, lock during scan + lock = self.lock() + for name, size in self.store.walk(): + entries.append((name, size)) + total_bytes += size + return entries, total_bytes + finally: + del lock + # used to avoid circular references so destructors work def aftertrans(files): renamefiles = [tuple(t) for t in files]
--- a/mercurial/statichttprepo.py Thu Jul 24 16:32:51 2008 +0200 +++ b/mercurial/statichttprepo.py Thu Jul 24 16:32:52 2008 +0200 @@ -55,14 +55,13 @@ # setup store if "store" in requirements: - self.encodefn = store.encodefilename - self.decodefn = store.decodefilename self.spath = self.path + "/store" else: - self.encodefn = lambda x: x - self.decodefn = lambda x: x self.spath = self.path - self.sopener = store.encodedopener(opener(self.spath), self.encodefn) + self.encodefn = store.encodefn(requirements) + so = opener(self.spath) + self.sopener = lambda path, *args, **kw: so( + self.encodefn(path), *args, **kw) self.manifest = manifest.manifest(self.sopener) self.changelog = changelog.changelog(self.sopener)
--- a/mercurial/store.py Thu Jul 24 16:32:51 2008 +0200 +++ b/mercurial/store.py Thu Jul 24 16:32:52 2008 +0200 @@ -5,6 +5,8 @@ # This software may be used and distributed according to the terms # of the GNU General Public License, incorporated herein by reference. +import os, stat, osutil, util + def _buildencodefun(): e = '_' win_reserved = [ord(x) for x in '\\:*?"<>|'] @@ -33,7 +35,91 @@ encodefilename, decodefilename = _buildencodefun() -def encodedopener(openerfn, fn): - def o(path, *args, **kw): - return openerfn(fn(path), *args, **kw) - return o +def _dirwalk(path, recurse): + '''yields (filename, size)''' + for e, kind, st in osutil.listdir(path, stat=True): + pe = os.path.join(path, e) + if kind == stat.S_IFDIR: + if recurse: + for x in _dirwalk(pe, True): + yield x + elif kind == stat.S_IFREG: + yield pe, st.st_size + +class _store: + '''base class for local repository stores''' + def __init__(self, path): + self.path = path + try: + # files in .hg/ will be created using this mode + mode = os.stat(self.path).st_mode + # avoid some useless chmods + if (0777 & ~util._umask) == (0777 & mode): + mode = None + except OSError: + mode = None + self.createmode = mode + + def join(self, f): + return os.path.join(self.path, f) + + def _revlogfiles(self, relpath='', recurse=False): + '''yields (filename, size)''' + if relpath: + path = os.path.join(self.path, relpath) + else: + path = self.path + striplen = len(self.path) + len(os.sep) + filetypes = ('.d', '.i') + for f, size in _dirwalk(path, recurse): + if (len(f) > 2) and f[-2:] in filetypes: + yield util.pconvert(f[striplen:]), size + + def _datafiles(self): + for x in self._revlogfiles('data', True): + yield x + + def walk(self): + '''yields (direncoded filename, size)''' + # yield data files first + for x in self._datafiles(): + yield x + # yield manifest before changelog + meta = util.sort(self._revlogfiles()) + meta.reverse() + for x in meta: + yield x + +class directstore(_store): + def __init__(self, path): + _store.__init__(self, path) + self.encodefn = lambda x: x + self.opener = util.opener(self.path) + self.opener.createmode = self.createmode + +class encodedstore(_store): + def __init__(self, path): + _store.__init__(self, os.path.join(path, 'store')) + self.encodefn = encodefilename + op = util.opener(self.path) + op.createmode = self.createmode + self.opener = lambda f, *args, **kw: op(self.encodefn(f), *args, **kw) + + def _datafiles(self): + for f, size in self._revlogfiles('data', True): + yield decodefilename(f), size + + def join(self, f): + return os.path.join(self.path, self.encodefn(f)) + +def encodefn(requirements): + if 'store' not in requirements: + return lambda x: x + else: + return encodefilename + +def store(requirements, path): + if 'store' not in requirements: + return directstore(path) + else: + return encodedstore(path)
--- a/mercurial/streamclone.py Thu Jul 24 16:32:51 2008 +0200 +++ b/mercurial/streamclone.py Thu Jul 24 16:32:52 2008 +0200 @@ -5,40 +5,12 @@ # This software may be used and distributed according to the terms # of the GNU General Public License, incorporated herein by reference. -import os, osutil, stat, util, lock +import util, lock # if server supports streaming clone, it advertises "stream" # capability with value that is version+flags of repo it is serving. # client only streams if it can read that repo format. -def walkrepo(root): - '''iterate over metadata files in repository. - walk in natural (sorted) order. - yields 2-tuples: name of .d or .i file, size of file.''' - - strip_count = len(root) + len(os.sep) - def walk(path, recurse): - for e, kind, st in osutil.listdir(path, stat=True): - pe = os.path.join(path, e) - if kind == stat.S_IFDIR: - if recurse: - for x in walk(pe, True): - yield x - else: - if kind != stat.S_IFREG or len(e) < 2: - continue - sfx = e[-2:] - if sfx in ('.d', '.i'): - yield pe[strip_count:], st.st_size - # write file data first - for x in walk(os.path.join(root, 'data'), True): - yield x - # write manifest before changelog - meta = util.sort(walk(root, False)) - meta.reverse() - for x in meta: - yield x - # stream file format is simple. # # server writes out line that says how many files, how many total @@ -59,28 +31,14 @@ fileobj.write('1\n') return - # get consistent snapshot of repo. lock during scan so lock not - # needed while we stream, and commits can happen. - repolock = None try: - try: - repolock = repo.lock() - except (lock.LockHeld, lock.LockUnavailable), inst: - repo.ui.warn('locking the repository failed: %s\n' % (inst,)) - fileobj.write('2\n') - return + entries, total_bytes = repo.storefiles() + except (lock.LockHeld, lock.LockUnavailable), inst: + repo.ui.warn('locking the repository failed: %s\n' % (inst,)) + fileobj.write('2\n') + return - fileobj.write('0\n') - repo.ui.debug('scanning\n') - entries = [] - total_bytes = 0 - for name, size in walkrepo(repo.spath): - name = repo.decodefn(util.pconvert(name)) - entries.append((name, size)) - total_bytes += size - finally: - del repolock - + fileobj.write('0\n') repo.ui.debug('%d files, %d bytes to transfer\n' % (len(entries), total_bytes)) fileobj.write('%d %d\n' % (len(entries), total_bytes))