Mercurial > evolve
changeset 4030:9871809aa348 mercurial-4.6
test-compat: merge stable into mercurial-4.6
| author | Pierre-Yves David <pierre-yves.david@octobus.net> |
|---|---|
| date | Tue, 28 Aug 2018 11:24:49 +0200 |
| parents | 85c99079d41f (current diff) e8ce05aa9138 (diff) |
| children | 9a2db13b2e99 eb4d07a0b19f |
| files | tests/test-discovery-obshashrange.t |
| diffstat | 8 files changed, 163 insertions(+), 68 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgtags Tue Aug 21 13:25:28 2018 +0200 +++ b/.hgtags Tue Aug 28 11:24:49 2018 +0200 @@ -68,3 +68,4 @@ 116cdd8c102ab0ae6295fb4886b0882e75e4d8f7 8.0.0 0887c30255a1a1808d74a63b16e896d457f8ef32 8.0.1 2c5d79c6459c6fabe0eb8723fc5041ac0dac7a9a 8.1.0 +e7abf863e1130e14cd4d65e53467a199d267b4fd 8.1.1
--- a/CHANGELOG Tue Aug 21 13:25:28 2018 +0200 +++ b/CHANGELOG Tue Aug 28 11:24:49 2018 +0200 @@ -1,7 +1,12 @@ Changelog ========= -8.1.1 - in progress +8.1.2 - in progress +------------------- + + * obshashrange: improved robusness of the cache under heavy load + +8.1.1 -- 2018-08-21 ------------------- * clone: fix possible crash when using clone bundle and forcing cache warming
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/hammerclient.py Tue Aug 28 11:24:49 2018 +0200 @@ -0,0 +1,27 @@ +#!/usr/bin/env python +import os +import sys +import subprocess + +if len(sys.argv) < 2: + execname = os.path.basename(sys.argv[0]) + print >> sys.stderr, "usage: %s CLIENT_ID" % execname + +client_id = sys.argv[1] + +subprocess.check_call(['hg', 'branch', "--force", "hammer-branch-%s" % client_id]) + +while True: + subprocess.check_call([ + 'hg', 'commit', + "--config", "ui.allowemptycommit=yes", + "--message", "hammer-%s" % client_id, + ]) + nodeid = subprocess.check_output([ + 'hg', 'log', '--rev', '.', '--template', '{node}' + ]) + subprocess.check_call([ + 'hg', 'debugobsolete', ''.join(reversed(nodeid)), nodeid + ]) + subprocess.check_call(['hg', 'pull']) + subprocess.check_call(['hg', 'push', '--force'])
--- a/debian/changelog Tue Aug 21 13:25:28 2018 +0200 +++ b/debian/changelog Tue Aug 28 11:24:49 2018 +0200 @@ -1,3 +1,9 @@ +mercurial-evolve (8.1.1) unstable; urgency=medium + + * new upstream release + + -- Pierre-Yves David <pierre-yves.david@ens-lyon.org> Tue, 21 Aug 2018 15:28:29 +0200 + mercurial-evolve (8.0.1-1) unstable; urgency=medium * New upstream release
--- a/hgext3rd/evolve/metadata.py Tue Aug 21 13:25:28 2018 +0200 +++ b/hgext3rd/evolve/metadata.py Tue Aug 28 11:24:49 2018 +0200 @@ -5,7 +5,7 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. -__version__ = '8.1.1.dev' +__version__ = '8.1.2.dev' testedwith = '4.3.2 4.4.2 4.5.2 4.6.2 4.7' minimumhgversion = '4.3' buglink = 'https://bz.mercurial-scm.org/'
--- a/hgext3rd/evolve/obsdiscovery.py Tue Aug 21 13:25:28 2018 +0200 +++ b/hgext3rd/evolve/obsdiscovery.py Tue Aug 28 11:24:49 2018 +0200 @@ -328,6 +328,11 @@ ### sqlite caching _sqliteschema = [ + """CREATE TABLE obshashrange(rev INTEGER NOT NULL, + idx INTEGER NOT NULL, + obshash BLOB NOT NULL, + PRIMARY KEY(rev, idx));""", + "CREATE INDEX range_index ON obshashrange(rev, idx);", """CREATE TABLE meta(schemaversion INTEGER NOT NULL, tiprev INTEGER NOT NULL, tipnode BLOB NOT NULL, @@ -335,11 +340,6 @@ obssize BLOB NOT NULL, obskey BLOB NOT NULL );""", - """CREATE TABLE obshashrange(rev INTEGER NOT NULL, - idx INTEGER NOT NULL, - obshash BLOB NOT NULL, - PRIMARY KEY(rev, idx));""", - "CREATE INDEX range_index ON obshashrange(rev, idx);", ] _queryexist = "SELECT name FROM sqlite_master WHERE type='table' AND name='meta';" _clearmeta = """DELETE FROM meta;""" @@ -432,10 +432,17 @@ value = self._data.get(rangeid) if value is None and self._con is not None: nrange = (rangeid[0], rangeid[1]) - obshash = self._con.execute(_queryobshash, nrange).fetchone() - if obshash is not None: - value = obshash[0] - self._data[rangeid] = value + try: + obshash = self._con.execute(_queryobshash, nrange).fetchone() + if obshash is not None: + value = obshash[0] + self._data[rangeid] = value + except (sqlite3.DatabaseError, sqlite3.OperationalError): + # something is wrong with the sqlite db + # Since this is a cache, we ignore it. + if '_con' in vars(self): + del self._con + self._new.clear() return value def __setitem__(self, rangeid, obshash): @@ -473,14 +480,19 @@ if con is not None: # always reset for now, the code detecting affect is buggy # so we need to reset more broadly than we would like. - if repo.stablerange._con is None: - con.execute(_reset) - self._data.clear() - else: - ranges = repo.stablerange.contains(repo, affected) - con.executemany(_delete, ranges) - for r in ranges: - self._data.pop(r, None) + try: + if repo.stablerange._con is None: + con.execute(_reset) + self._data.clear() + else: + ranges = repo.stablerange.contains(repo, affected) + con.executemany(_delete, ranges) + for r in ranges: + self._data.pop(r, None) + except (sqlite3.DatabaseError, sqlite3.OperationalError) as exc: + repo.ui.log('evoext-cache', 'error while updating obshashrange cache: %s' % exc) + del self._updating + return # rewarm key revisions # @@ -524,7 +536,7 @@ util.makedirs(self._vfs.dirname(self._path)) except OSError: return None - con = sqlite3.connect(self._path) + con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE") con.text_factory = str return con @@ -568,6 +580,21 @@ repo.ui.warn(msg) def _save(self, repo): + if not self._new: + return + try: + return self._trysave(repo) + except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc: + # Catch error that may arise under stress + # + # operational error catch read-only and locked database + # IntegrityError catch Unique constraint error that may arise + if '_con' in vars(self): + del self._con + self._new.clear() + repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc) + + def _trysave(self, repo): if self._con is None: util.unlinkpath(self._path, ignoremissing=True) if '_con' in vars(self): @@ -582,30 +609,34 @@ for req in _sqliteschema: con.execute(req) - con.execute(_newmeta, self._fullcachekey) + meta = [self._schemaversion] + list(self.emptykey) + con.execute(_newmeta, meta) + self._ondiskcachekey = self.emptykey else: con = self._con - if self._ondiskcachekey is not None: - meta = con.execute(_querymeta).fetchone() - if meta[1:] != self._ondiskcachekey: - # drifting is currently an issue because this means another - # process might have already added the cache line we are about - # to add. This will confuse sqlite - msg = _('obshashrange cache: skipping write, ' - 'database drifted under my feet\n') - data = (meta[2], meta[1], self._ondiskcachekey[0], self._ondiskcachekey[1]) - repo.ui.warn(msg) - return - data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new) - con.executemany(_updateobshash, data) - cachekey = self._fullcachekey - con.execute(_clearmeta) # remove the older entry - con.execute(_newmeta, cachekey) - con.commit() - self._new.clear() - self._ondiskcachekey = self._cachekey - self._valid = True - + with con: + meta = con.execute(_querymeta).fetchone() + if meta[1:] != self._ondiskcachekey: + # drifting is currently an issue because this means another + # process might have already added the cache line we are about + # to add. This will confuse sqlite + msg = _('obshashrange cache: skipping write, ' + 'database drifted under my feet\n') + repo.ui.warn(msg) + self._new.clear() + self._valid = False + if '_con' in vars(self): + del self._con + self._valid = False + return + data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new) + con.executemany(_updateobshash, data) + cachekey = self._fullcachekey + con.execute(_clearmeta) # remove the older entry + con.execute(_newmeta, cachekey) + self._new.clear() + self._valid = True + self._ondiskcachekey = self._cachekey @eh.wrapfunction(obsolete.obsstore, '_addmarkers') def _addmarkers(orig, obsstore, *args, **kwargs): obsstore.rangeobshashcache.clear()
--- a/hgext3rd/evolve/stablerangecache.py Tue Aug 21 13:25:28 2018 +0200 +++ b/hgext3rd/evolve/stablerangecache.py Tue Aug 28 11:24:49 2018 +0200 @@ -17,6 +17,7 @@ from mercurial import ( error, localrepo, + node as nodemod, util, ) @@ -87,10 +88,6 @@ ############################# _sqliteschema = [ - """CREATE TABLE meta(schemaversion INTEGER NOT NULL, - tiprev INTEGER NOT NULL, - tipnode BLOB NOT NULL - );""", """CREATE TABLE range(rev INTEGER NOT NULL, idx INTEGER NOT NULL, PRIMARY KEY(rev, idx));""", @@ -106,6 +103,10 @@ "CREATE INDEX subranges_index ON subranges (suprev, supidx);", "CREATE INDEX superranges_index ON subranges (subrev, subidx);", "CREATE INDEX range_index ON range (rev, idx);", + """CREATE TABLE meta(schemaversion INTEGER NOT NULL, + tiprev INTEGER NOT NULL, + tipnode BLOB NOT NULL + );""", ] _newmeta = "INSERT INTO meta (schemaversion, tiprev, tipnode) VALUES (?,?,?);" _updatemeta = "UPDATE meta SET tiprev = ?, tipnode = ?;" @@ -177,11 +178,19 @@ cache = self._subrangescache if rangeid not in cache and rangeid[0] <= self._ondisktiprev and self._con is not None: value = None - result = self._con.execute(_queryrange, rangeid).fetchone() - if result is not None: # database know about this node (skip in the future?) - value = self._con.execute(_querysubranges, rangeid).fetchall() - # in memory caching of the value - cache[rangeid] = value + try: + result = self._con.execute(_queryrange, rangeid).fetchone() + if result is not None: # database know about this node (skip in the future?) + value = self._con.execute(_querysubranges, rangeid).fetchall() + # in memory caching of the value + cache[rangeid] = value + except (sqlite3.DatabaseError, sqlite3.OperationalError): + # something is wrong with the sqlite db + # Since this is a cache, we ignore it. + if '_con' in vars(self): + del self._con + self._unsavedsubranges.clear() + return cache.get(rangeid) def _setsub(self, rangeid, value): @@ -194,7 +203,7 @@ util.makedirs(self._vfs.dirname(self._path)) except OSError: return None - con = sqlite3.connect(self._path) + con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE") con.text_factory = str return con @@ -223,6 +232,21 @@ return con def _save(self, repo): + if not self._unsavedsubranges: + return + try: + return self._trysave(repo) + except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc: + # Catch error that may arise under stress + # + # operational error catch read-only and locked database + # IntegrityError catch Unique constraint error that may arise + if '_con' in vars(self): + del self._con + self._unsavedsubranges.clear() + repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc) + + def _trysave(self, repo): repo = repo.unfiltered() repo.depthcache.save(repo) repo.stablesort.save(repo) @@ -238,19 +262,20 @@ con = self._db() if con is None: return - con.execute('BEGIN IMMEDIATE;') with con: for req in _sqliteschema: con.execute(req) meta = [self._schemaversion, - self._tiprev, - self._tipnode, + nodemod.nullrev, + nodemod.nullid, ] con.execute(_newmeta, meta) + self._ondisktiprev = nodemod.nullrev + self._ondisktipnode = nodemod.nullid else: con = self._con - con.execute('BEGIN IMMEDIATE;') + with con: meta = con.execute(_querymeta).fetchone() if meta[2] != self._ondisktipnode or meta[1] != self._ondisktiprev: # drifting is currently an issue because this means another @@ -258,19 +283,19 @@ # to add. This will confuse sqlite msg = _('stable-range cache: skipping write, ' 'database drifted under my feet\n') - hint = _('(disk: %s-%s vs mem: %s%s)\n') - data = (meta[2], meta[1], self._ondisktiprev, self._ondisktipnode) + hint = _('(disk: %s-%s vs mem: %s-%s)\n') + data = (nodemod.hex(meta[2]), meta[1], + nodemod.hex(self._ondisktipnode), self._ondisktiprev) repo.ui.warn(msg) repo.ui.warn(hint % data) - con.execute('ROLLBACK;') + self._unsavedsubranges.clear() return - meta = [self._tiprev, - self._tipnode, - ] - con.execute(_updatemeta, meta) - - self._saverange(con, repo) - con.commit() + else: + self._saverange(con, repo) + meta = [self._tiprev, + self._tipnode, + ] + con.execute(_updatemeta, meta) self._ondisktiprev = self._tiprev self._ondisktipnode = self._tipnode self._unsavedsubranges.clear()
--- a/tests/test-discovery-obshashrange.t Tue Aug 21 13:25:28 2018 +0200 +++ b/tests/test-discovery-obshashrange.t Tue Aug 28 11:24:49 2018 +0200 @@ -445,8 +445,8 @@ * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obscache in *.???? seconds (0r, 1o) (glob) * @0000000000000000000000000000000000000000 (*)> -R ../server debugobsolete bbbbbbb2222222222bbbbbbbbbbbbb2222222222 bebd167eb94d257ace0e814aeb98e6972ed2970d exited 0 after *.?? seconds (glob) * @0000000000000000000000000000000000000000 (*)> -R server serve --stdio (glob) - 1970/01/01 00:00:00 * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 2 changeset and cached ranges (glob) - * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 2o) (glob) + 1970/01/01 00:00:00 * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 3 changeset and cached ranges (glob) + * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 4o) (glob) * @0000000000000000000000000000000000000000 (*)> -R server serve --stdio exited 0 after *.?? seconds (glob) * @0000000000000000000000000000000000000000 (*)> -R ../server blackbox (glob) $ rm ../server/.hg/blackbox.log