Mercurial > evolve
changeset 224:f60555898df4
new obsolete store
The new formats allows multiple "new" changeset and various meta-data.
Forcing extinct to secret phase happen now on an irregular basis
A command exist to migrate to the new format
| author | Pierre-Yves.David@ens-lyon.org |
|---|---|
| date | Tue, 08 May 2012 20:50:43 +0200 |
| parents | d254ca935aa5 |
| children | 72af5aa43997 |
| files | docs/tutorials/tutorial.t enable.sh hgext/obsolete.py tests/test-oldconvert.t |
| diffstat | 4 files changed, 344 insertions(+), 135 deletions(-) [+] |
line wrap: on
line diff
--- a/docs/tutorials/tutorial.t Sun May 06 03:27:45 2012 +0200 +++ b/docs/tutorials/tutorial.t Tue May 08 20:50:43 2012 +0200 @@ -494,6 +494,7 @@ added 1 changesets with 1 changes to 1 files (+1 heads) (run 'hg heads .' to see heads, 'hg merge' to merge) 1 new unstables changesets + $ hg log -r 'extinct()' $ hg log 9ac5d0e790a2 (draft): animals ffa278c50818 (draft): bathroom stuff
--- a/enable.sh Sun May 06 03:27:45 2012 +0200 +++ b/enable.sh Tue May 08 20:50:43 2012 +0200 @@ -26,7 +26,8 @@ obsolete=XXXREPOPATHXXX/hgext/obsolete.py # history rewriting UI -hgext.rebase= # needed by evolve +# needed by evolve +hgext.rebase= evolve=XXXREPOPATHXXX/hgext/evolve.py
--- a/hgext/obsolete.py Sun May 06 03:27:45 2012 +0200 +++ b/hgext/obsolete.py Tue May 08 20:50:43 2012 +0200 @@ -83,6 +83,7 @@ from mercurial.i18n import _ import base64 +import json from mercurial import util from mercurial import context @@ -163,9 +164,10 @@ """obsolete parents of a subset""" cs = set() nm = repo.changelog.nodemap + markerbysubj = repo.obsoletestore.subjects for r in s: - for p in repo._obssubrels.get(repo[r].node(), ()): - pr = nm.get(p) + for p in markerbysubj.get(repo[r].node(), ()): + pr = nm.get(p['object']) if pr is not None: cs.add(pr) return cs @@ -180,9 +182,11 @@ """obsolete ancestors of a subset""" toproceed = [repo[r].node() for r in s] seen = set() + allsubjects = repo.obsoletestore.subjects while toproceed: nc = toproceed.pop() - for np in repo._obssubrels.get(nc, ()): + for mark in allsubjects.get(nc, ()): + np = mark['object'] if np not in seen: seen.add(np) toproceed.append(np) @@ -253,30 +257,25 @@ def pushobsolete(repo, key, old, raw): """push obsolete relation through pushkey""" - assert key == "relations" - w = repo.wlock() + assert key == "markers" + l = repo.lock() try: tmp = StringIO() tmp.write(raw) tmp.seek(0) - relations = _obsdeserialise(tmp) - for sub, objs in relations.iteritems(): - for obj in objs: - try: - repo.addobsolete(sub, obj) - except error.RepoLookupError: - pass - return 0 + repo.obsoletestore.load(tmp) + repo.obsoletestore._dirty = True # XXX meh + return 1 finally: - w.release() + l.release() def listobsolete(repo): """dump all obsolete relation in XXX this have be improved""" tmp = StringIO() - _obsserialise(repo._obssubrels, tmp) - return {'relations': base64.b64encode(tmp.getvalue())} + repo.obsoletestore.save(tmp) + return {'markers': base64.b64encode(tmp.getvalue())} pushkey.register('obsolete', pushobsolete, listobsolete) @@ -345,11 +344,12 @@ new = set() while nodes: n = nodes.pop() - if n in repo._obsobjrels: - newernodes = repo._obsobjrels[n] - for newernode in newernodes: - if newernode is not None: - nodes.append(newernode) + if n in repo.obsoletestore.objects: + markers = repo.obsoletestore.objects[n] + for mark in markers: + for newernode in mark['subjects']: + if newernode is not None: + nodes.append(newernode) else: new.add(n) if new: @@ -384,12 +384,53 @@ """Add an obsolete relation between a too node The subject is expected to be a newer version of the object""" - sub = repo[subject] - obj = repo[object] - repo.addobsolete(sub.node(), obj.node()) + lock = repo.lock() + try: + sub = repo[subject] + obj = repo[object] + repo.addobsolete(sub.node(), obj.node()) + finally: + lock.release() return 0 -cmdtable = {'debugobsolete': (cmddebugobsolete, [], '<subject> <object>')} +def cmddebugconvertobsolete(ui, repo): + cnt = 0 + l = repo.lock() + try: + repo._importoldobsolete = True + store = repo.obsoletestore + try: + f = repo.opener('obsolete-relations') + try: + for line in f: + subhex, objhex = line.split() + sub = bin(subhex) + obj = bin(objhex) + newmarker = { + 'subjects': (sub==nullid) and [] or [sub], + 'object': obj, + 'date': util.makedate(), + 'user': ui.username(), + 'reason': 'import from older format.', + } + store.new(newmarker) + store._dirty = True + cnt += 1 + finally: + f.close() + util.unlink(repo.join('obsolete-relations')) + except IOError: + ui.warn('nothing to do\n') + pass + finally: + del repo._importoldobsolete + l.release() + ui.status('%i obsolete marker converted\n' % cnt) + + +cmdtable = {'debugobsolete': (cmddebugobsolete, [], '<subject> <object>'), + 'debugconvertobsolete': (cmddebugconvertobsolete, [], ''), + } ### Altering existing command ############################# @@ -400,11 +441,16 @@ ui.warn(_('Working directory parent is obsolete\n')) return res +def noextinctsvisibleheads(orig, repo): + repo._turn_extinct_secret() + return orig(repo) + def uisetup(ui): extensions.wrapcommand(commands.table, "update", wrapmayobsoletewc) extensions.wrapcommand(commands.table, "pull", wrapmayobsoletewc) extensions.wrapfunction(discovery, 'findcommonoutgoing', wrapfindcommonoutgoing) extensions.wrapfunction(discovery, 'checkheads', wrapcheckheads) + extensions.wrapfunction(phases, 'visibleheads', noextinctsvisibleheads) if util.safehasattr(phases, 'visiblebranchmap'): extensions.wrapfunction(phases, 'visiblebranchmap', wrapvisiblebranchmap) @@ -447,30 +493,128 @@ def newerversion(repo, obs): """Return the newer version of an obsolete changeset""" - toproceed = set([obs]) - # XXX know optimization available + toproceed = set([(obs,)]) + # XXX known optimization available newer = set() + objectrels = repo.obsoletestore.objects while toproceed: current = toproceed.pop() - if current in repo._obsobjrels: - toproceed.update(repo._obsobjrels[current]) - elif current is not None: # None is kill - newer.add((current,)) + assert len(current) <= 1, 'splitting not handled yet. %r' % current + if current: + n, = current + if n in objectrels: + markers = objectrels[n] + for mark in markers: + toproceed.add(tuple(mark['subjects'])) + else: + newer.add(tuple(current)) else: newer.add(()) return sorted(newer) +### obsolete relation storage +############################# +def add2set(d, key, mark): + """add <mark> to a `set` in <d>[<key>]""" + d.setdefault(key, []).append(mark) + +def markerid(marker): + KEYS = ['subjects', "object", "date", "user", "reason"] + for key in KEYS: + assert key in marker + keys = sorted(marker.keys()) + a = util.sha1() + for key in keys: + if key == 'subjects': + for sub in sorted(marker[key]): + a.update(sub) + elif key == 'id': + pass + else: + a.update(str(marker[key])) + a.update('\0') + return a.digest() + +class obsoletestore(object): + """Store obsolete relations + + Relation are stored in three mapping. All mapping have "obsolete markers" + as values:: + + {'id': "unique id of the obsolete marker" + 'subjects': "0-N newer version of changeset in "object" (as ordered list) + 'object': "old and obsolete version" + 'date': "When was this marker created ?" + 'user': "Who did that ?" + 'reason': "Why was it done" + } + + Three keys exists + + :self._markers: "id" -> marker + + :self.subjects: "subject" -> set(marker) + + :self.objects: "object" -> set(marker) + """ + + def __init__(self): + self._markers = {} + self.subjects = {} + self.objects = {} + self._dirty = False # should be on repo + + def new(self, marker): + """Add a *new* marker to the store. computing it's ID""" + mid = marker['id'] = markerid(marker) + self._insert(marker) + self._dirty = True + return mid + + def _insert(self, marker): + if marker['id'] not in self._markers: + self._markers[marker['id']] = marker + add2set(self.objects, marker['object'], marker) + for subj in marker['subjects']: + add2set(self.subjects, subj, marker) + + def save(self, stream): + markers = [] + for mark in self._markers.itervalues(): + jmark = mark.copy() + jmark['id'] = hex(jmark['id']) + jmark['subjects'] = [hex(n) for n in jmark['subjects']] + jmark['object'] = hex(jmark['object']) + markers.append(jmark) + json.dump(markers, stream) + + def load(self, stream): + for mark in json.load(stream): + mark['id'] = bin(mark['id']) + mark['subjects'] = [bin(n) for n in mark['subjects']] + mark['object'] = bin(mark['object']) + self._insert(mark) + +def writeobsolete(repo): + """wire obsolete data on disk""" + f = repo.sopener('obsoletemarkers', 'w', atomictemp=True) + try: + repo.obsoletestore.save(f) + repo._dirty = False + finally: + f.close() + ### repo subclassing ############################# def reposetup(ui, repo): - if not repo.local(): return opull = repo.pull opush = repo.push + olock = repo.lock o_rollback = repo._rollback o_updatebranchcache = repo.updatebranchcache @@ -485,18 +629,39 @@ ### Public method def obsoletedby(self, node): """return the set of node that make <node> obsolete (obj)""" - return self._obsobjrels.get(node, set()) + others = set() + for marker in self.obsoletestore.objects.get(node, []): + others.update(marker['subjects']) + return others def obsolete(self, node): """return the set of node that <node> make obsolete (sub)""" - return self._obssubrels.get(node, set()) + return set(marker['object'] for marker in self.obsoletestore.subjects.get(node, [])) + + @util.propertycache + def obsoletestore(self): + if not getattr(self, '_importoldobsolete', False): + try: + f = self.opener('obsolete-relations') + f.close() + raise util.Abort('old format of obsolete marker detected!\n' + 'run `hg debugconvertobsolete` once.') + except IOError: + pass + store = obsoletestore() + try: + f = self.sopener('obsoletemarkers') + store.load(f) + except IOError: + pass + return store @util.propertycache def _obsoleteset(self): """the set of obsolete revision""" obs = set() nm = self.changelog.nodemap - for obj in self._obsobjrels: + for obj in self.obsoletestore.objects: try: # /!\api change in Hg 2.2 (e8d37b78acfb22ae2c1fb126c2)/!\ rev = nm.get(obj) except TypeError: #XXX to remove while breaking Hg 2.1 support @@ -521,10 +686,6 @@ return set(self.revs('obsolete() - obsolete()::unstable()')) def _clearobsoletecache(self): - if '_obsobjrels' in vars(self): - del self._obsobjrels - if '_obssubrels' in vars(self): - del self._obssubrels if '_obsoleteset' in vars(self): del self._obsoleteset self._clearunstablecache() @@ -543,55 +704,56 @@ def addobsolete(self, sub, obj): """Add a relation marking that node <sub> is a new version of <obj>""" - if sub == nullid: - sub = None - if obj in self._obssubrels.get(sub, set()): - return 0 - if sub == obj: - return 0 - self._obssubrels.setdefault(sub, set()).add(obj) - self._obsobjrels.setdefault(obj, set()).add(sub) + assert sub != obj + if not repo[obj].phase(): + if sub is None: + self.ui.warn( + _("trying to kill immutable changeset %(obj)s\n") + % {'obj': short(obj)}) + if sub is not None: + self.ui.warn( + _("%(sub)s try to obsolete immutable changeset %(obj)s\n") + % {'sub': short(sub), 'obj': short(obj)}) + lock = self.lock() try: - if self[obj].phase() == 0: - if sub is None: - self.ui.warn( - _("trying to kill immutable changeset %(obj)s\n") - % {'obj': short(obj)}) - if sub is not None: - self.ui.warn( - _("%(sub)s try to obsolete immutable changeset %(obj)s\n") - % {'sub': short(sub), 'obj': short(obj)}) - self.changelog.hiddenrevs.add(repo[obj].rev()) - except (error.RepoLookupError, error.LookupError): - pass #unknow revision (but keep propagating the data - self._writeobsrels() + newmarker = { + 'subjects': (sub==nullid) and [] or [sub], + 'object': obj, + 'date': util.makedate(), + 'user': ui.username(), + 'reason': 'unknown', + } + mid = self.obsoletestore.new(newmarker) + self._clearobsoletecache() + self._turn_extinct_secret() + return mid + finally: + lock.release() + + def _turn_extinct_secret(self): + """ensure all extinct changeset are secret""" self._clearobsoletecache() - return 1 - - ### obsolete storage - @util.propertycache - def _obsobjrels(self): - """{<old-node> -> set(<new-node>)} - - also compute hidden revision""" - #reverse sub -> objs mapping - objrels = {} - for sub, objs in self._obssubrels.iteritems(): - for obj in objs: - objrels.setdefault(obj, set()).add(sub) - return objrels - - @util.propertycache - def _obssubrels(self): - """{<new-node> -> set(<old-node>)}""" - return self._readobsrels() - - - + # this is mainly for safety purpose + # both pull and push + expobs = [c.node() for c in repo.set('extinct() - secret()')] + phases.retractboundary(repo, 2, expobs) ### Disk IO + + def lock(self, *args, **kwargs): + l = olock(*args, **kwargs) + if not getattr(l.releasefn, 'obspatched', False): + oreleasefn = l.releasefn + def releasefn(*args, **kwargs): + if self.obsoletestore._dirty: + writeobsolete(self) + oreleasefn(*args, **kwargs) + releasefn.obspatched = True + l.releasefn = releasefn + return l + def _readobsrels(self): - """Write obsolete relation on disk""" + """Read obsolete relation on disk""" # XXX handle lock try: f = self.opener('obsolete-relations') @@ -602,47 +764,36 @@ except IOError: return {} - def _writeobsrels(self): - """Write obsolete relation on disk""" - # XXX handle lock - lock = self.wlock() - try: - f = self.opener('obsolete-relations', 'w', atomictemp=True) - try: - _obsserialise(self._obssubrels, f) - try: - f.rename() - except AttributeError: # old version - f.close() - finally: - f.close() - finally: - lock.release() - ### pull // push support def pull(self, remote, *args, **kwargs): """wrapper around push that push obsolete relation""" - result = opull(remote, *args, **kwargs) - if 'obsolete' in remote.listkeys('namespaces'): - tmp = StringIO() - rels = remote.listkeys('obsolete')['relations'] - tmp.write(base64.b64decode(rels)) - tmp.seek(0) - obsrels = _obsdeserialise(tmp) - for sub, objs in obsrels.iteritems(): - for obj in objs: - self.addobsolete(sub, obj) - return result + l = repo.lock() + try: + result = opull(remote, *args, **kwargs) + if 'obsolete' in remote.listkeys('namespaces'): + tmp = StringIO() + rels = remote.listkeys('obsolete')['markers'] + tmp.write(base64.b64decode(rels)) + tmp.seek(0) + repo.obsoletestore.load(tmp) + repo.obsoletestore._dirty = True # XXX meh + self._clearobsoletecache() + self._turn_extinct_secret() + return result + finally: + l.release() def push(self, remote, *args, **opts): """wrapper around pull that pull obsolete relation""" + self._turn_extinct_secret() result = opush(remote, *args, **opts) if 'obsolete' in remote.listkeys('namespaces'): tmp = StringIO() - _obsserialise(self._obssubrels, tmp) - remote.pushkey('obsolete', 'relations', '', tmp.getvalue()) + self.obsoletestore.save(tmp) + remote.pushkey('obsolete', 'markers', '', tmp.getvalue()) + self._turn_extinct_secret() return result @@ -652,27 +803,27 @@ # /!\ api change in Hg 2.2 (97efd26eb9576f39590812ea9) /!\ if util.safehasattr(repo, '_journalfiles'): # Hg 2.2 def _journalfiles(self): - return o_journalfiles() + (self.join('journal.obsolete-relations'),) + return o_journalfiles() + (self.sjoin('journal.obsoletemarkers'),) def _writejournal(self, desc): """wrapped version of _writejournal that save obsolete data""" o_writejournal(desc) - filename = 'obsolete-relations' - filepath = self.join(filename) + filename = 'obsoletemarkers' + filepath = self.sjoin(filename) if os.path.exists(filepath): journalname = 'journal.' + filename - journalpath = self.join(journalname) + journalpath = self.sjoin(journalname) util.copyfile(filepath, journalpath) else: # XXX remove this bloc while breaking support to Hg 2.1 def _writejournal(self, desc): """wrapped version of _writejournal that save obsolete data""" entries = list(o_writejournal(desc)) - filename = 'obsolete-relations' - filepath = self.join(filename) + filename = 'obsoletemarkers' + filepath = self.sjoin(filename) if os.path.exists(filepath): journalname = 'journal.' + filename - journalpath = self.join(journalname) + journalpath = self.sjoin(journalname) util.copyfile(filepath, journalpath) entries.append(journalpath) return tuple(entries) @@ -681,16 +832,15 @@ """wrapped version of _rollback that restore obsolete data""" ret = o_rollback(dryrun, force) if not (ret or dryrun): #rollback did not failed - src = self.join('undo.obsolete-relations') - dst = self.join('obsolete-relations') + src = self.sjoin('undo.obsoletemarkers') + dst = self.sjoin('obsoletemarkers') if os.path.exists(src): util.rename(src, dst) elif os.path.exists(dst): # If no state was saved because the file did not existed before. os.unlink(dst) # invalidate cache - self.__dict__.pop('_obssubrels', None) - self.__dict__.pop('_obsobjrels', None) + self.__dict__.pop('obsoletestore', None) return ret @storecache('00changelog.i') @@ -721,17 +871,16 @@ c.__class__ = hchangelog return c - - - repo.__class__ = obsoletingrepo - expobs = [c.node() for c in repo.set('extinct() - secret()')] - if expobs: # do not lock in nothing move. locking for peanut make hgview reload on any command - lock = repo.lock() - try: - phases.retractboundary(repo, 2, expobs) - finally: - lock.release() + if False: + expobs = [c.node() for c in repo.set('extinct() - secret()')] + if expobs: # do not lock in nothing move. locking for peanut make hgview reload on any command + lock = repo.lock() + try: + expobs = [c.node() for c in repo.set('extinct() - secret()')] + phases.retractboundary(repo, 2, expobs) + finally: + lock.release()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test-oldconvert.t Tue May 08 20:50:43 2012 +0200 @@ -0,0 +1,58 @@ + $ cat >> $HGRCPATH <<EOF + > [web] + > push_ssl = false + > allow_push = * + > [phases] + > publish=False + > [alias] + > odiff=diff --rev 'limit(obsparents(.),1)' --rev . + > [extensions] + > hgext.graphlog= + > EOF + $ mkcommit() { + > echo "$1" > "$1" + > hg add "$1" + > hg ci -m "add $1" + > } + +create commit + + $ hg init repo + $ cd repo + $ mkcommit a + $ mkcommit b + $ hg up -q 0 + $ mkcommit c + created new head + +forge old style relation files + + $ hg log -r 2 --template='{node} ' > .hg/obsolete-relations + $ hg log -r 1 --template='{node}' >> .hg/obsolete-relations + +enable the extensions + + $ echo "obsolete=$(echo $(dirname $TESTDIR))/hgext/obsolete.py" >> $HGRCPATH + + $ hg glog + abort: old format of obsolete marker detected! + run `hg debugconvertobsolete` once. + [255] + $ hg debugconvertobsolete --traceback + 1 obsolete marker converted + $ hg glog + @ changeset: 2:d67cd0334eee + | tag: tip + | parent: 0:1f0dee641bb7 + | user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: add c + | + o changeset: 0:1f0dee641bb7 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + summary: add a + + $ hg debugconvertobsolete + nothing to do + 0 obsolete marker converted