Mercurial > evolve
changeset 334:fb83210bce32 stable
obsolete: move to official binary format
| author | Pierre-Yves David <pierre-yves.david@logilab.fr> |
|---|---|
| date | Tue, 03 Jul 2012 12:58:55 +0200 |
| parents | 36e2016d6563 |
| children | c27a465daef2 aab826129142 |
| files | docs/tutorials/tutorial.t hgext/obsolete.py tests/test-oldconvert.t tests/test-qsync.t |
| diffstat | 4 files changed, 313 insertions(+), 157 deletions(-) [+] |
line wrap: on
line diff
--- a/docs/tutorials/tutorial.t Wed Jun 27 17:21:24 2012 +0200 +++ b/docs/tutorials/tutorial.t Tue Jul 03 12:58:55 2012 +0200 @@ -223,7 +223,7 @@ adding manifests adding file changes added 1 changesets with 1 changes to 1 files (+1 heads) - (run 'hg heads .' to see heads, 'hg merge' to merge) + (run 'hg update' to get a working copy) I now have a new heads. Note that this remote head is immutable
--- a/hgext/obsolete.py Wed Jun 27 17:21:24 2012 +0200 +++ b/hgext/obsolete.py Tue Jul 03 12:58:55 2012 +0200 @@ -86,6 +86,12 @@ import base64 import json +import struct +from mercurial import util, base85 + +_pack = struct.pack +_unpack = struct.unpack + from mercurial import util from mercurial import context from mercurial import revset @@ -191,10 +197,10 @@ """Precursor of a changeset""" cs = set() nm = repo.changelog.nodemap - markerbysubj = repo.obsoletestore.subjects + markerbysubj = repo.obsstore.successors for r in s: for p in markerbysubj.get(repo[r].node(), ()): - pr = nm.get(p['object']) + pr = nm.get(p[0]) if pr is not None: cs.add(pr) return cs @@ -209,11 +215,11 @@ """transitive precursors of a subset""" toproceed = [repo[r].node() for r in s] seen = set() - allsubjects = repo.obsoletestore.subjects + allsubjects = repo.obsstore.successors while toproceed: nc = toproceed.pop() for mark in allsubjects.get(nc, ()): - np = mark['object'] + np = mark[0] if np not in seen: seen.add(np) toproceed.append(np) @@ -235,10 +241,10 @@ """Successors of a changeset""" cs = set() nm = repo.changelog.nodemap - markerbyobj = repo.obsoletestore.objects + markerbyobj = repo.obsstore.precursors for r in s: for p in markerbyobj.get(repo[r].node(), ()): - for sub in p['subjects']: + for sub in p[1]: sr = nm.get(sub) if sr is not None: cs.add(sr) @@ -254,11 +260,11 @@ """transitive successors of a subset""" toproceed = [repo[r].node() for r in s] seen = set() - allobjects = repo.obsoletestore.objects + allobjects = repo.obsstore.precursors while toproceed: nc = toproceed.pop() for mark in allobjects.get(nc, ()): - for sub in mark['subjects']: + for sub in mark[1]: if sub not in seen: seen.add(sub) toproceed.append(sub) @@ -407,29 +413,30 @@ # Pushkey mechanism for mutable ######################################### -def pushobsolete(repo, key, old, raw): - """push obsolete relation through pushkey""" - assert key == "markers" - l = repo.lock() +def listmarkers(repo): + """List markers over pushkey""" + if not repo.obsstore: + return {} + data = repo.obsstore._writemarkers() + return {'dump': base85.b85encode(data)} + +def pushmarker(repo, key, old, new): + """Push markers over pushkey""" + if key != 'dump': + repo.ui.warn(_('unknown key: %r') % key) + return 0 + if old: + repo.ui.warn(_('unexpected old value') % key) + return 0 + data = base85.b85decode(new) + lock = repo.lock() try: - tmp = StringIO() - tmp.write(raw) - tmp.seek(0) - repo.obsoletestore.load(tmp) - repo.obsoletestore._dirty = True # XXX meh + repo.obsstore.mergemarkers(data) return 1 finally: - l.release() - -def listobsolete(repo): - """dump all obsolete relation in + lock.release() - XXX this have be improved""" - tmp = StringIO() - repo.obsoletestore.save(tmp) - return {'markers': base64.b64encode(tmp.getvalue())} - -pushkey.register('obsolete', pushobsolete, listobsolete) +pushkey.register('obsolete', pushmarker, listmarkers) ### Discovery wrapping ############################# @@ -499,10 +506,10 @@ new = set() while nodes: n = nodes.pop() - if n in repo.obsoletestore.objects: - markers = repo.obsoletestore.objects[n] + if n in repo.obsstore.precursors: + markers = repo.obsstore.precursors[n] for mark in markers: - for newernode in mark['subjects']: + for newernode in mark[1]: if newernode is not None: nodes.append(newernode) else: @@ -563,35 +570,51 @@ """import markers from an .hg/obsolete-relations file""" cnt = 0 l = repo.lock() + some = False try: repo._importoldobsolete = True - store = repo.obsoletestore + store = repo.obsstore + ### very first format try: f = repo.opener('obsolete-relations') try: + some = True for line in f: subhex, objhex = line.split() - sub = bin(subhex) - obj = bin(objhex) - newmarker = { - 'subjects': (sub==nullid) and [] or [sub], - 'object': obj, - 'date': util.makedate(), + suc = bin(subhex) + prec = bin(objhex) + sucs = (suc==nullid) and [] or [suc] + meta = { + 'date': '%i %i' % util.makedate(), 'user': ui.username(), - 'reason': 'import from older format.', } - store.new(newmarker) - store._dirty = True + store.create(prec, sucs, 0, meta) cnt += 1 finally: f.close() util.unlink(repo.join('obsolete-relations')) except IOError: - ui.warn('nothing to do\n') pass + ### second (json) format + data = repo.sopener.tryread('obsoletemarkers') + if data: + some = True + for oldmark in json.loads(data): + del oldmark['id'] # dropped for now + del oldmark['reason'] # unused until then + oldmark['subjects'] = [bin(n) for n in oldmark['subjects']] + oldmark['object'] = bin(oldmark['object']) + oldmark['date'] = '%i %i' % tuple(oldmark['date']) + store.create(oldmark.pop('object'), + oldmark.pop('subjects'), + 0, oldmark) + cnt += 1 + util.unlink(repo.sjoin('obsoletemarkers')) finally: del repo._importoldobsolete l.release() + if not some: + ui.warn('nothing to do\n') ui.status('%i obsolete marker converted\n' % cnt) @command('debugsuccessors', [], '') @@ -603,9 +626,9 @@ """ lock = repo.lock() try: - allsuccessors = repo.obsoletestore.objects + allsuccessors = repo.obsstore.precursors for old in sorted(allsuccessors): - successors = [sorted(m['subjects']) for m in allsuccessors[old]] + successors = [sorted(m[1]) for m in allsuccessors[old]] for i, group in enumerate(sorted(successors)): ui.write('%s' % short(old)) for new in group: @@ -633,14 +656,14 @@ if new != oldnode: lock = repo.lock() try: - newmarker = { + meta = { 'subjects': [new], 'object': oldnode, 'date': util.makedate(), 'user': ui.username(), 'reason': 'commit --amend', } - repo.obsoletestore.new(newmarker) + repo.obsstore.create(oldnode, [new], 0, meta) repo._clearobsoletecache() repo._turn_extinct_secret() finally: @@ -701,7 +724,7 @@ toproceed = set([(obs,)]) # XXX known optimization available newer = set() - objectrels = repo.obsoletestore.objects + objectrels = repo.obsstore.precursors while toproceed: current = toproceed.pop() assert len(current) <= 1, 'splitting not handled yet. %r' % current @@ -710,7 +733,7 @@ if n in objectrels: markers = objectrels[n] for mark in markers: - toproceed.add(tuple(mark['subjects'])) + toproceed.add(tuple(mark[1])) else: newer.add(tuple(current)) else: @@ -740,74 +763,158 @@ a.update('\0') return a.digest() -class obsoletestore(object): - """Store obsolete relations +# mercurial backport + +def encodemeta(meta): + """Return encoded metadata string to string mapping. + + Assume no ':' in key and no '\0' in both key and value.""" + for key, value in meta.iteritems(): + if ':' in key or '\0' in key: + raise ValueError("':' and '\0' are forbidden in metadata key'") + if '\0' in value: + raise ValueError("':' are forbidden in metadata value'") + return '\0'.join(['%s:%s' % (k, meta[k]) for k in sorted(meta)]) - Relation are stored in three mapping. All mapping have "obsolete markers" - as values:: +def decodemeta(data): + """Return string to string dictionary from encoded version.""" + d = {} + for l in data.split('\0'): + if l: + key, value = l.split(':') + d[key] = value + return d + +# data used for parsing and writing +_fmversion = 0 +_fmfixed = '>BIB20s' +_fmnode = '20s' +_fmfsize = struct.calcsize(_fmfixed) +_fnodesize = struct.calcsize(_fmnode) - {'id': "unique id of the obsolete marker" - 'subjects': "0-N newer version of changeset in "object" (as ordered list) - 'object': "old and obsolete version" - 'date': "When was this marker created ?" - 'user': "Who did that ?" - 'reason': "Why was it done" - } +def _readmarkers(data): + """Read and enumerate markers from raw data""" + off = 0 + diskversion = _unpack('>B', data[off:off + 1])[0] + off += 1 + if diskversion != _fmversion: + raise util.Abort(_('parsing obsolete marker: unknown version %r') + % diskversion) - Three keys exists - - :self._markers: "id" -> marker + # Loop on markers + l = len(data) + while off + _fmfsize <= l: + # read fixed part + cur = data[off:off + _fmfsize] + off += _fmfsize + nbsuc, mdsize, flags, pre = _unpack(_fmfixed, cur) + # read replacement + sucs = () + if nbsuc: + s = (_fnodesize * nbsuc) + cur = data[off:off + s] + sucs = _unpack(_fmnode * nbsuc, cur) + off += s + # read metadata + # (metadata will be decoded on demand) + metadata = data[off:off + mdsize] + if len(metadata) != mdsize: + raise util.Abort(_('parsing obsolete marker: metadata is too ' + 'short, %d bytes expected, got %d') + % (len(metadata), mdsize)) + off += mdsize + yield (pre, sucs, flags, metadata) - :self.subjects: "subject" -> set(marker) +class obsstore(object): + """Store obsolete markers - :self.objects: "object" -> set(marker) + Markers can be accessed with two mappings: + - precursors: old -> set(new) + - successors: new -> set(old) """ def __init__(self): - self._markers = {} - self.subjects = {} - self.objects = {} - self._dirty = False # should be on repo + self._all = [] + # new markers to serialize + self._new = [] + self.precursors = {} + self.successors = {} + + def __iter__(self): + return iter(self._all) + + def __nonzero__(self): + return bool(self._all) + + def create(self, prec, succs=(), flag=0, metadata=None): + """obsolete: add a new obsolete marker - def new(self, marker): - """Add a *new* marker to the store. computing it's ID""" - mid = marker['id'] = markerid(marker) - self._insert(marker) - self._dirty = True - return mid + * ensuring it is hashable + * check mandatory metadata + * encode metadata + """ + if metadata is None: + metadata = {} + if len(prec) != 20: + raise ValueError(prec) + for succ in succs: + if len(succ) != 20: + raise ValueError(prec) + marker = (str(prec), tuple(succs), int(flag), encodemeta(metadata)) + self.add(marker) - def _insert(self, marker): - if marker['id'] not in self._markers: - self._markers[marker['id']] = marker - add2set(self.objects, marker['object'], marker) - for subj in marker['subjects']: - add2set(self.subjects, subj, marker) + def add(self, marker): + """Add a new marker to the store + + This marker still needs to be written to disk""" + self._new.append(marker) + self._load(marker) + + def loadmarkers(self, data): + """Load all markers in data, mark them as known.""" + for marker in _readmarkers(data): + self._load(marker) - def save(self, stream): - markers = [] - for mark in self._markers.itervalues(): - jmark = mark.copy() - jmark['id'] = hex(jmark['id']) - jmark['subjects'] = [hex(n) for n in jmark['subjects']] - jmark['object'] = hex(jmark['object']) - markers.append(jmark) - json.dump(markers, stream, indent=4) + def mergemarkers(self, data): + other = set(_readmarkers(data)) + local = set(self._all) + new = other - local + for marker in new: + self.add(marker) + + def flushmarkers(self, stream): + """Write all markers to a stream + + After this operation, "new" markers are considered "known".""" + self._writemarkers(stream) + self._new[:] = [] - def load(self, stream): - for mark in json.load(stream): - mark['id'] = bin(mark['id']) - mark['subjects'] = [bin(n) for n in mark['subjects']] - mark['object'] = bin(mark['object']) - self._insert(mark) + def _load(self, marker): + self._all.append(marker) + pre, sucs = marker[:2] + self.precursors.setdefault(pre, set()).add(marker) + for suc in sucs: + self.successors.setdefault(suc, set()).add(marker) -def writeobsolete(repo): - """wire obsolete data on disk""" - f = repo.sopener('obsoletemarkers', 'w', atomictemp=True) - try: - repo.obsoletestore.save(f) - repo._dirty = False - finally: - f.close() + def _writemarkers(self, stream=None): + # Kept separate from flushmarkers(), it will be reused for + # markers exchange. + if stream is None: + final = [] + w = final.append + else: + w = stream.write + w(_pack('>B', _fmversion)) + for marker in self._all: + pre, sucs, flags, metadata = marker + nbsuc = len(sucs) + format = _fmfixed + (_fmnode * nbsuc) + data = [nbsuc, len(metadata), flags, pre] + data.extend(sucs) + w(_pack(format, *data)) + w(metadata) + if stream is None: + return ''.join(final) ### repo subclassing @@ -835,30 +942,27 @@ def obsoletedby(self, node): """return the set of node that make <node> obsolete (obj)""" others = set() - for marker in self.obsoletestore.objects.get(node, []): - others.update(marker['subjects']) + for marker in self.obsstore.precursors.get(node, []): + others.update(marker[1]) return others def obsolete(self, node): """return the set of node that <node> make obsolete (sub)""" - return set(marker['object'] for marker in self.obsoletestore.subjects.get(node, [])) + return set(marker[0] for marker in self.obsstore.successors.get(node, [])) - @util.propertycache - def obsoletestore(self): + @storecache('obsstore') + def obsstore(self): if not getattr(self, '_importoldobsolete', False): - try: - f = self.opener('obsolete-relations') - f.close() + data = repo.opener.tryread('obsolete-relations') + if not data: + data = repo.sopener.tryread('obsoletemarkers') + if data: raise util.Abort('old format of obsolete marker detected!\n' 'run `hg debugconvertobsolete` once.') - except IOError: - pass - store = obsoletestore() - try: - f = self.sopener('obsoletemarkers') - store.load(f) - except IOError: - pass + store = obsstore() + data = self.sopener.tryread('obsstore') + if data: + store.loadmarkers(data) return store @util.propertycache @@ -866,7 +970,7 @@ """the set of obsolete revision""" obs = set() nm = self.changelog.nodemap - for obj in self.obsoletestore.objects: + for obj in self.obsstore.precursors: try: # /!\api change in Hg 2.2 (e8d37b78acfb22ae2c1fb126c2)/!\ rev = nm.get(obj) except TypeError: #XXX to remove while breaking Hg 2.1 support @@ -929,14 +1033,13 @@ % {'sub': short(sub), 'obj': short(obj)}) lock = self.lock() try: - newmarker = { - 'subjects': (sub==nullid) and [] or [sub], - 'object': obj, + meta = { 'date': util.makedate(), 'user': ui.username(), 'reason': 'unknown', } - mid = self.obsoletestore.new(newmarker) + subs = (sub == nullid) and [] or [sub] + mid = self.obsstore.create(obj, subs, 0, meta) self._clearobsoletecache() self._turn_extinct_secret() return mid @@ -968,25 +1071,19 @@ if not getattr(l.releasefn, 'obspatched', False): oreleasefn = l.releasefn def releasefn(*args, **kwargs): - if self.obsoletestore._dirty: - writeobsolete(self) + if 'obsstore' in vars(self) and self.obsstore._new: + f = self.sopener('obsstore', 'wb', atomictemp=True) + try: + self.obsstore.flushmarkers(f) + f.close() + except: # re-raises + f.discard() + raise oreleasefn(*args, **kwargs) releasefn.obspatched = True l.releasefn = releasefn return l - def _readobsrels(self): - """Read obsolete relation on disk""" - # XXX handle lock - try: - f = self.opener('obsolete-relations') - try: - return _obsdeserialise(f) - finally: - f.close() - except IOError: - return {} - ### pull // push support @@ -995,16 +1092,13 @@ l = repo.lock() try: result = opull(remote, *args, **kwargs) - if 'obsolete' in remote.listkeys('namespaces'): - tmp = StringIO() - rels = remote.listkeys('obsolete')['markers'] - tmp.write(base64.b64decode(rels)) - tmp.seek(0) - repo.obsoletestore.load(tmp) - repo.obsoletestore._dirty = True # XXX meh + remoteobs = remote.listkeys('obsolete') + if 'dump' in remoteobs: + data = base85.b85decode(remoteobs['dump']) + self.obsstore.mergemarkers(data) self._clearobsoletecache() - self._turn_extinct_secret() - return result + self._turn_extinct_secret() + return result finally: l.release() @@ -1012,10 +1106,12 @@ """wrapper around pull that pull obsolete relation""" self._turn_extinct_secret() result = opush(remote, *args, **opts) - if 'obsolete' in remote.listkeys('namespaces'): - tmp = StringIO() - self.obsoletestore.save(tmp) - remote.pushkey('obsolete', 'markers', '', tmp.getvalue()) + if 'obsolete' in self.listkeys('namespaces') and self.obsstore: + data = self.obsstore._writemarkers() + r = remote.pushkey('obsolete', 'dump', '', + base85.b85encode(data)) + if not r: + self.ui.warn(_('failed to push obsolete markers!\n')) self._turn_extinct_secret() return result @@ -1026,12 +1122,12 @@ # /!\ api change in Hg 2.2 (97efd26eb9576f39590812ea9) /!\ if util.safehasattr(repo, '_journalfiles'): # Hg 2.2 def _journalfiles(self): - return o_journalfiles() + (self.sjoin('journal.obsoletemarkers'),) + return o_journalfiles() + (self.sjoin('journal.obsstore'),) def _writejournal(self, desc): """wrapped version of _writejournal that save obsolete data""" o_writejournal(desc) - filename = 'obsoletemarkers' + filename = 'obsstore' filepath = self.sjoin(filename) if os.path.exists(filepath): journalname = 'journal.' + filename @@ -1042,7 +1138,7 @@ def _writejournal(self, desc): """wrapped version of _writejournal that save obsolete data""" entries = list(o_writejournal(desc)) - filename = 'obsoletemarkers' + filename = 'obsstore' filepath = self.sjoin(filename) if os.path.exists(filepath): journalname = 'journal.' + filename @@ -1055,15 +1151,15 @@ """wrapped version of _rollback that restore obsolete data""" ret = o_rollback(dryrun, force) if not (ret or dryrun): #rollback did not failed - src = self.sjoin('undo.obsoletemarkers') - dst = self.sjoin('obsoletemarkers') + src = self.sjoin('undo.obsstore') + dst = self.sjoin('obsstore') if os.path.exists(src): util.rename(src, dst) elif os.path.exists(dst): # If no state was saved because the file did not existed before. os.unlink(dst) # invalidate cache - self.__dict__.pop('obsoletestore', None) + self.__dict__.pop('obsstore', None) return ret @storecache('00changelog.i')
--- a/tests/test-oldconvert.t Wed Jun 27 17:21:24 2012 +0200 +++ b/tests/test-oldconvert.t Tue Jul 03 12:58:55 2012 +0200 @@ -53,6 +53,65 @@ date: Thu Jan 01 00:00:00 1970 +0000 summary: add a + $ hg debugsuccessors + 7c3bad9141dc d67cd0334eee $ hg debugconvertobsolete nothing to do 0 obsolete marker converted + +Convert json + + $ cat > .hg/store/obsoletemarkers << EOF + > [ + > { + > "reason": "import from older format.", + > "subjects": [ + > "3218406b50ed13480765e7c260669620f37fba6e" + > ], + > "user": "Pierre-Yves David <pierre-yves.david@ens-lyon.org>", + > "date": [ + > 1336503323.9768269, + > -7200 + > ], + > "object": "3e03d82708d4da97a92158558dd13386d8f09ad5", + > "id": "4743f676eaf3923cb98c921ee06b2e91052c365b" + > }, + > { + > "reason": "import from older format.", + > "subjects": [ + > "a37154106de358121a32a3dd6ba0281931550515" + > ], + > "user": "Pierre-Yves David <pierre-yves.david@logilab.fr>", + > "date": [ + > 1336557472.7875929, + > -7200 + > ], + > "object": "5c722672795c3a2cb94d0cc9a821c394c1475f87", + > "id": "1fd90a84b7225d2e3062b7e1b3100aa2e060fc72" + > }, + > { + > "reason": "import from older format.", + > "subjects": [ + > "dc8d8aa6e8ac490fcafd074d570df0787ee0ee69" + > ], + > "user": "Pierre-Yves David <pierre-yves.david@logilab.fr>", + > "date": [ + > 1336557472.784307, + > -7200 + > ], + > "object": "2c3784e102bb34ccc93862af5bd6d609ee30c577", + > "id": "7d940c5ee1f886c8a6c0d805b43e522cb3ef7a15" + > } + > ] + > EOF + $ hg glog + abort: old format of obsolete marker detected! + run `hg debugconvertobsolete` once. + [255] + $ hg debugconvertobsolete --traceback + 3 obsolete marker converted + $ hg debugsuccessors + 2c3784e102bb dc8d8aa6e8ac + 3e03d82708d4 3218406b50ed + 5c722672795c a37154106de3 + 7c3bad9141dc d67cd0334eee
--- a/tests/test-qsync.t Wed Jun 27 17:21:24 2012 +0200 +++ b/tests/test-qsync.t Tue Jul 03 12:58:55 2012 +0200 @@ -182,6 +182,7 @@ pulling from ../local2 searching for changes no changes found + (run 'hg update' to get a working copy) $ hg pull --mq ../local2/.hg/patches pulling from ../local2/.hg/patches searching for changes