Mercurial > hg
view tests/testlib/persistent-nodemap-race-ext.py @ 49603:3eda36e9b3d6 stable
matcher: fix issues regex flag contained in pattern (issue6759)
Python 3.11 is now enforcing that flag must be at the beginning of the regex
This creates a serious regression for people using Python 3.11 with an hgignore
using flag in a "relre" pattern.
We now detect any flags in such pattern and "prepend" our ".*" pattern after them.
In addition, we now insert the flag in the regexp to only affect the pattern we
are rewriting. Otherwise, the regex built from the combined pattern would these
flags in the middle of it anyway.
As a side effect of this last change, we fix a bug… before this change regex
flag in a pattern would affect all combined patterns. That was bad and is not
longer the case.
The Rust code needs to be updated to fix that very bug, but we will do it in
another changeset.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Wed, 16 Nov 2022 13:05:01 +0100 |
parents | 6000f5b25c9b |
children | 805d4a462abb |
line wrap: on
line source
"""Create the race condition for issue6554 The persistent nodemap issues had an issue where a second writer could overwrite the data that a previous write just wrote. The would break the append only garantee of the persistent nodemap and could confuse reader. This extensions create all the necessary synchronisation point to the race condition to happen. It involves 3 process <LEFT> (a writer) <RIGHT> (a writer) and <READER> [1] <LEFT> take the lock and start a transaction [2] <LEFT> updated `00changelog.i` with the new data [3] <RIGHT> reads: - the new changelog index `00changelog.i` - the old `00changelog.n` [4] <LEFT> update the persistent nodemap: - writing new data from the last valid offset - updating the docket (00changelog.n) [5] <LEFT> release the lock [6] <RIGHT> grab the lock and run `repo.invalidate` [7] <READER> reads: - the changelog index after <LEFT> write - the nodemap docket after <LEFT> write [8] <RIGHT> reload the changelog since `00changelog.n` changed /!\ This is the faulty part in issue 6554, the outdated docket is kept [9] <RIGHT> write: - the changelog index (00changelog.i) - the nodemap data (00changelog*.nd) /!\ if the outdated docket is used, the write starts from the same ofset /!\ as in [4], overwriting data that <LEFT> wrote in step [4]. - the nodemap docket (00changelog.n) [10] <READER> reads the nodemap data from `00changelog*.nd` /!\ if step [9] was wrong, the data matching the docket that <READER> /!\ loaded have been overwritten and the expected root-nodes is no longer /!\ valid. """ import os from mercurial.revlogutils.constants import KIND_CHANGELOG from mercurial import ( changelog, encoding, extensions, localrepo, node, pycompat, registrar, testing, util, ) from mercurial.revlogutils import ( nodemap as nodemaputil, ) configtable = {} configitem = registrar.configitem(configtable) configitem(b'devel', b'nodemap-race.role', default=None) cmdtable = {} command = registrar.command(cmdtable) LEFT = b'left' RIGHT = b'right' READER = b'reader' SYNC_DIR = os.path.join(encoding.environ[b'TESTTMP'], b'sync-files') # mark the end of step [1] FILE_LEFT_LOCKED = os.path.join(SYNC_DIR, b'left-locked') # mark that step [3] is ready to run. FILE_RIGHT_READY_TO_LOCK = os.path.join(SYNC_DIR, b'right-ready-to-lock') # mark the end of step [2] FILE_LEFT_CL_DATA_WRITE = os.path.join(SYNC_DIR, b'left-data') # mark the end of step [4] FILE_LEFT_CL_NODEMAP_WRITE = os.path.join(SYNC_DIR, b'left-nodemap') # mark the end of step [3] FILE_RIGHT_CL_NODEMAP_READ = os.path.join(SYNC_DIR, b'right-nodemap') # mark that step [9] is read to run FILE_RIGHT_CL_NODEMAP_PRE_WRITE = os.path.join( SYNC_DIR, b'right-pre-nodemap-write' ) # mark that step [9] has run. FILE_RIGHT_CL_NODEMAP_POST_WRITE = os.path.join( SYNC_DIR, b'right-post-nodemap-write' ) # mark that step [7] is ready to run FILE_READER_READY = os.path.join(SYNC_DIR, b'reader-ready') # mark that step [7] has run FILE_READER_READ_DOCKET = os.path.join(SYNC_DIR, b'reader-read-docket') def _print(*args, **kwargs): print(*args, **kwargs) def _role(repo): """find the role associated with the process""" return repo.ui.config(b'devel', b'nodemap-race.role') def wrap_changelog_finalize(orig, cl, tr): """wrap the update of `00changelog.i` during transaction finalization This is useful for synchronisation before or after the file is updated on disk. """ role = getattr(tr, '_race_role', None) if role == RIGHT: print('right ready to write, waiting for reader') testing.wait_file(FILE_READER_READY) testing.write_file(FILE_RIGHT_CL_NODEMAP_PRE_WRITE) testing.wait_file(FILE_READER_READ_DOCKET) print('right proceeding with writing its changelog index and nodemap') ret = orig(cl, tr) print("finalized changelog write") if role == LEFT: testing.write_file(FILE_LEFT_CL_DATA_WRITE) return ret def wrap_persist_nodemap(orig, tr, revlog, *args, **kwargs): """wrap the update of `00changelog.n` and `*.nd` during tr finalization This is useful for synchronisation before or after the files are updated on disk. """ is_cl = revlog.target[0] == KIND_CHANGELOG role = getattr(tr, '_race_role', None) if is_cl: if role == LEFT: testing.wait_file(FILE_RIGHT_CL_NODEMAP_READ) if is_cl: print("persisting changelog nodemap") print(" new data start at", revlog._nodemap_docket.data_length) ret = orig(tr, revlog, *args, **kwargs) if is_cl: print("persisted changelog nodemap") print_nodemap_details(revlog) if role == LEFT: testing.write_file(FILE_LEFT_CL_NODEMAP_WRITE) elif role == RIGHT: testing.write_file(FILE_RIGHT_CL_NODEMAP_POST_WRITE) return ret def print_nodemap_details(cl): """print relevant information about the nodemap docket currently in memory""" dkt = cl._nodemap_docket print('docket-details:') if dkt is None: print(' <no-docket>') return print(' uid: ', pycompat.sysstr(dkt.uid)) print(' actual-tip: ', cl.tiprev()) print(' tip-rev: ', dkt.tip_rev) print(' data-length:', dkt.data_length) def wrap_persisted_data(orig, revlog): """print some information about the nodemap information we just read Used by the <READER> process only. """ ret = orig(revlog) if ret is not None: docket, data = ret file_path = nodemaputil._rawdata_filepath(revlog, docket) file_path = revlog.opener.join(file_path) file_size = os.path.getsize(file_path) print('record-data-length:', docket.data_length) print('actual-data-length:', len(data)) print('file-actual-length:', file_size) return ret def sync_read(orig): """used by <READER> to force the race window This make sure we read the docker from <LEFT> while reading the datafile after <RIGHT> write. """ orig() testing.write_file(FILE_READER_READ_DOCKET) print('reader: nodemap docket read') testing.wait_file(FILE_RIGHT_CL_NODEMAP_POST_WRITE) def uisetup(ui): class RacedRepo(localrepo.localrepository): def lock(self, wait=True): # make sure <RIGHT> as the "Wrong" information in memory before # grabbing the lock newlock = self._currentlock(self._lockref) is None if newlock and _role(self) == LEFT: cl = self.unfiltered().changelog print_nodemap_details(cl) elif newlock and _role(self) == RIGHT: testing.write_file(FILE_RIGHT_READY_TO_LOCK) print('nodemap-race: right side start of the locking sequence') testing.wait_file(FILE_LEFT_LOCKED) testing.wait_file(FILE_LEFT_CL_DATA_WRITE) self.invalidate(clearfilecache=True) print('nodemap-race: right side reading changelog') cl = self.unfiltered().changelog tiprev = cl.tiprev() tip = cl.node(tiprev) tiprev2 = cl.rev(tip) if tiprev != tiprev2: raise RuntimeError( 'bad tip -round-trip %d %d' % (tiprev, tiprev2) ) testing.write_file(FILE_RIGHT_CL_NODEMAP_READ) print('nodemap-race: right side reading of changelog is done') print_nodemap_details(cl) testing.wait_file(FILE_LEFT_CL_NODEMAP_WRITE) print('nodemap-race: right side ready to wait for the lock') ret = super(RacedRepo, self).lock(wait=wait) if newlock and _role(self) == LEFT: print('nodemap-race: left side locked and ready to commit') testing.write_file(FILE_LEFT_LOCKED) testing.wait_file(FILE_RIGHT_READY_TO_LOCK) cl = self.unfiltered().changelog print_nodemap_details(cl) elif newlock and _role(self) == RIGHT: print('nodemap-race: right side locked and ready to commit') cl = self.unfiltered().changelog print_nodemap_details(cl) return ret def transaction(self, *args, **kwargs): # duck punch the role on the transaction to help other pieces of code tr = super(RacedRepo, self).transaction(*args, **kwargs) tr._race_role = _role(self) return tr localrepo.localrepository = RacedRepo extensions.wrapfunction( nodemaputil, 'persist_nodemap', wrap_persist_nodemap ) extensions.wrapfunction( changelog.changelog, '_finalize', wrap_changelog_finalize ) def reposetup(ui, repo): if _role(repo) == READER: extensions.wrapfunction( nodemaputil, 'persisted_data', wrap_persisted_data ) extensions.wrapfunction(nodemaputil, 'test_race_hook_1', sync_read) class ReaderRepo(repo.__class__): @util.propertycache def changelog(self): print('reader ready to read the changelog, waiting for right') testing.write_file(FILE_READER_READY) testing.wait_file(FILE_RIGHT_CL_NODEMAP_PRE_WRITE) return super(ReaderRepo, self).changelog repo.__class__ = ReaderRepo @command(b'check-nodemap-race') def cmd_check_nodemap_race(ui, repo): """Run proper <READER> access in the race Windows and check nodemap content""" repo = repo.unfiltered() print('reader: reading changelog') cl = repo.changelog print('reader: changelog read') print_nodemap_details(cl) tip_rev = cl.tiprev() tip_node = cl.node(tip_rev) print('tip-rev: ', tip_rev) print('tip-node:', node.short(tip_node).decode('ascii')) print('node-rev:', cl.rev(tip_node)) for r in cl.revs(): n = cl.node(r) try: r2 = cl.rev(n) except ValueError as exc: print('error while checking revision:', r) print(' ', exc) return 1 else: if r2 != r: print('revision %d is missing from the nodemap' % r) return 1