# HG changeset patch # User Pierre-Yves David # Date 1696982566 -7200 # Node ID 12c308c55e538415aaec1fa046c66deb9738ddb4 # Parent 752c5a5b73c687588ad0f52a7d9d3e0242709a06# Parent 704c3d0878d93fc4e8fe0b72b2b1c9f0f3fe91a9 branching: merge stable into default diff -r 752c5a5b73c6 -r 12c308c55e53 contrib/perf.py --- a/contrib/perf.py Wed Jan 25 15:34:27 2023 +0100 +++ b/contrib/perf.py Wed Oct 11 02:02:46 2023 +0200 @@ -882,23 +882,143 @@ fm.end() +def _default_clear_on_disk_tags_cache(repo): + from mercurial import tags + + repo.cachevfs.tryunlink(tags._filename(repo)) + + +def _default_clear_on_disk_tags_fnodes_cache(repo): + from mercurial import tags + + repo.cachevfs.tryunlink(tags._fnodescachefile) + + +def _default_forget_fnodes(repo, revs): + """function used by the perf extension to prune some entries from the + fnodes cache""" + from mercurial import tags + + missing_1 = b'\xff' * 4 + missing_2 = b'\xff' * 20 + cache = tags.hgtagsfnodescache(repo.unfiltered()) + for r in revs: + cache._writeentry(r * tags._fnodesrecsize, missing_1, missing_2) + cache.write() + + @command( b'perf::tags|perftags', formatteropts + [ (b'', b'clear-revlogs', False, b'refresh changelog and manifest'), + ( + b'', + b'clear-on-disk-cache', + False, + b'clear on disk tags cache (DESTRUCTIVE)', + ), + ( + b'', + b'clear-fnode-cache-all', + False, + b'clear on disk file node cache (DESTRUCTIVE),', + ), + ( + b'', + b'clear-fnode-cache-rev', + [], + b'clear on disk file node cache (DESTRUCTIVE),', + b'REVS', + ), + ( + b'', + b'update-last', + b'', + b'simulate an update over the last N revisions (DESTRUCTIVE),', + b'N', + ), ], ) def perftags(ui, repo, **opts): + """Benchmark tags retrieval in various situation + + The option marked as (DESTRUCTIVE) will alter the on-disk cache, possibly + altering performance after the command was run. However, it does not + destroy any stored data. + """ + from mercurial import tags + opts = _byteskwargs(opts) timer, fm = gettimer(ui, opts) repocleartagscache = repocleartagscachefunc(repo) clearrevlogs = opts[b'clear_revlogs'] + clear_disk = opts[b'clear_on_disk_cache'] + clear_fnode = opts[b'clear_fnode_cache_all'] + + clear_fnode_revs = opts[b'clear_fnode_cache_rev'] + update_last_str = opts[b'update_last'] + update_last = None + if update_last_str: + try: + update_last = int(update_last_str) + except ValueError: + msg = b'could not parse value for update-last: "%s"' + msg %= update_last_str + hint = b'value should be an integer' + raise error.Abort(msg, hint=hint) + + clear_disk_fn = getattr( + tags, + "clear_cache_on_disk", + _default_clear_on_disk_tags_cache, + ) + clear_fnodes_fn = getattr( + tags, + "clear_cache_fnodes", + _default_clear_on_disk_tags_fnodes_cache, + ) + clear_fnodes_rev_fn = getattr( + tags, + "forget_fnodes", + _default_forget_fnodes, + ) + + clear_revs = [] + if clear_fnode_revs: + clear_revs.extends(scmutil.revrange(repo, clear_fnode_revs)) + + if update_last: + revset = b'last(all(), %d)' % update_last + last_revs = repo.unfiltered().revs(revset) + clear_revs.extend(last_revs) + + from mercurial import repoview + + rev_filter = {(b'experimental', b'extra-filter-revs'): revset} + with repo.ui.configoverride(rev_filter, source=b"perf"): + filter_id = repoview.extrafilter(repo.ui) + + filter_name = b'%s%%%s' % (repo.filtername, filter_id) + pre_repo = repo.filtered(filter_name) + pre_repo.tags() # warm the cache + old_tags_path = repo.cachevfs.join(tags._filename(pre_repo)) + new_tags_path = repo.cachevfs.join(tags._filename(repo)) + + clear_revs = sorted(set(clear_revs)) def s(): + if update_last: + util.copyfile(old_tags_path, new_tags_path) if clearrevlogs: clearchangelog(repo) clearfilecache(repo.unfiltered(), 'manifest') + if clear_disk: + clear_disk_fn(repo) + if clear_fnode: + clear_fnodes_fn(repo) + elif clear_revs: + clear_fnodes_rev_fn(repo, clear_revs) repocleartagscache() def t(): diff -r 752c5a5b73c6 -r 12c308c55e53 hgext/blackbox.py --- a/hgext/blackbox.py Wed Jan 25 15:34:27 2023 +0100 +++ b/hgext/blackbox.py Wed Oct 11 02:02:46 2023 +0200 @@ -99,6 +99,7 @@ def _log(self, ui, event, msg, opts): default = ui.configdate(b'devel', b'default-date') dateformat = ui.config(b'blackbox', b'date-format') + debug_to_stderr = ui.configbool(b'blackbox', b'debug.to-stderr') if dateformat: date = dateutil.datestr(default, dateformat) else: @@ -130,7 +131,10 @@ maxfiles=self._maxfiles, maxsize=self._maxsize, ) as fp: - fp.write(fmt % args) + msg = fmt % args + fp.write(msg) + if debug_to_stderr: + ui.write_err(msg) except (IOError, OSError) as err: # deactivate this to avoid failed logging again self._trackedevents.clear() diff -r 752c5a5b73c6 -r 12c308c55e53 mercurial/bundle2.py --- a/mercurial/bundle2.py Wed Jan 25 15:34:27 2023 +0100 +++ b/mercurial/bundle2.py Wed Oct 11 02:02:46 2023 +0200 @@ -896,7 +896,7 @@ """utility to transfer a bundle2 as binary This is made necessary by the fact the 'getbundle' command over 'ssh' - have no way to know then the reply end, relying on the bundle to be + have no way to know when the reply ends, relying on the bundle to be interpreted to know its end. This is terrible and we are sorry, but we needed to move forward to get general delta enabled. """ diff -r 752c5a5b73c6 -r 12c308c55e53 mercurial/configitems.toml --- a/mercurial/configitems.toml Wed Jan 25 15:34:27 2023 +0100 +++ b/mercurial/configitems.toml Wed Oct 11 02:02:46 2023 +0200 @@ -2796,6 +2796,12 @@ [[items]] section = "blackbox" +name = "debug.to-stderr" +default = false +in_core_extension = "blackbox" + +[[items]] +section = "blackbox" name = "dirty" default = false in_core_extension = "blackbox" diff -r 752c5a5b73c6 -r 12c308c55e53 mercurial/httppeer.py --- a/mercurial/httppeer.py Wed Jan 25 15:34:27 2023 +0100 +++ b/mercurial/httppeer.py Wed Oct 11 02:02:46 2023 +0200 @@ -662,7 +662,8 @@ return inst except error.RepoError as httpexception: try: - r = statichttprepo.make_peer(ui, b"static-" + path.loc, create) + path = path.copy(new_raw_location=b"static-" + path.rawloc) + r = statichttprepo.make_peer(ui, path, create) ui.note(_(b'(falling back to static-http)\n')) return r except error.RepoError: diff -r 752c5a5b73c6 -r 12c308c55e53 mercurial/tags.py --- a/mercurial/tags.py Wed Jan 25 15:34:27 2023 +0100 +++ b/mercurial/tags.py Wed Oct 11 02:02:46 2023 +0200 @@ -190,10 +190,9 @@ _updatetags(cachetags, alltags) return alltags + has_node = repo.changelog.index.has_node for head in reversed(heads): # oldest to newest - assert repo.changelog.index.has_node( - head - ), b"tag cache returned bogus head %s" % short(head) + assert has_node(head), b"tag cache returned bogus head %s" % short(head) fnodes = _filterfnodes(tagfnode, reversed(heads)) alltags = _tagsfromfnodes(ui, repo, fnodes) @@ -910,3 +909,24 @@ ) finally: lock.release() + + +def clear_cache_on_disk(repo): + """function used by the perf extension to "tags" cache""" + repo.cachevfs.tryunlink(_filename(repo)) + + +def clear_cache_fnodes(repo): + """function used by the perf extension to clear "file node cache""" + repo.cachevfs.tryunlink(_filename(repo)) + + +def forget_fnodes(repo, revs): + """function used by the perf extension to prune some entries from the fnodes + cache""" + missing_1 = b'\xff' * 4 + missing_2 = b'\xff' * 20 + cache = hgtagsfnodescache(repo.unfiltered()) + for r in revs: + cache._writeentry(r * _fnodesrecsize, missing_1, missing_2) + cache.write() diff -r 752c5a5b73c6 -r 12c308c55e53 mercurial/upgrade_utils/actions.py --- a/mercurial/upgrade_utils/actions.py Wed Jan 25 15:34:27 2023 +0100 +++ b/mercurial/upgrade_utils/actions.py Wed Oct 11 02:02:46 2023 +0200 @@ -289,8 +289,7 @@ postdowngrademessage = _( b'repository downgraded to not use share safe mode, ' - b'existing shares will not work and needs to' - b' be reshared.' + b'existing shares will not work and need to be reshared.' ) postupgrademessage = _( @@ -359,7 +358,7 @@ description = _(b'Stores copies information alongside changesets.') upgrademessage = _( - b'Allows to use more efficient algorithm to deal with ' b'copy tracing.' + b'Allows to use more efficient algorithm to deal with copy tracing.' ) touches_filelogs = False diff -r 752c5a5b73c6 -r 12c308c55e53 rust/hg-core/src/revlog/mod.rs --- a/rust/hg-core/src/revlog/mod.rs Wed Jan 25 15:34:27 2023 +0100 +++ b/rust/hg-core/src/revlog/mod.rs Wed Oct 11 02:02:46 2023 +0200 @@ -236,6 +236,16 @@ data_path: Option<&Path>, use_nodemap: bool, ) -> Result { + Self::open_gen(store_vfs, index_path, data_path, use_nodemap, None) + } + + fn open_gen( + store_vfs: &Vfs, + index_path: impl AsRef, + data_path: Option<&Path>, + use_nodemap: bool, + nodemap_for_test: Option, + ) -> Result { let index_path = index_path.as_ref(); let index = { match store_vfs.mmap_open_opt(&index_path)? { @@ -273,6 +283,8 @@ ) }; + let nodemap = nodemap_for_test.or(nodemap); + Ok(Revlog { index, data_bytes, @@ -306,23 +318,13 @@ &self, node: NodePrefix, ) -> Result { - let looked_up = if let Some(nodemap) = &self.nodemap { + if let Some(nodemap) = &self.nodemap { nodemap .find_bin(&self.index, node)? .ok_or(RevlogError::InvalidRevision) } else { self.rev_from_node_no_persistent_nodemap(node) - }; - - if node.is_prefix_of(&NULL_NODE) { - return match looked_up { - Ok(_) => Err(RevlogError::AmbiguousPrefix), - Err(RevlogError::InvalidRevision) => Ok(NULL_REVISION), - res => res, - }; - }; - - looked_up + } } /// Same as `rev_from_node`, without using a persistent nodemap @@ -338,17 +340,23 @@ // TODO: consider building a non-persistent nodemap in memory to // optimize these cases. let mut found_by_prefix = None; - for rev in (0..self.len()).rev() { + for rev in (-1..self.len() as BaseRevision).rev() { let rev = Revision(rev as BaseRevision); - let index_entry = self.index.get_entry(rev).ok_or_else(|| { - HgError::corrupted( - "revlog references a revision not in the index", - ) - })?; - if node == *index_entry.hash() { + let candidate_node = if rev == Revision(-1) { + NULL_NODE + } else { + let index_entry = + self.index.get_entry(rev).ok_or_else(|| { + HgError::corrupted( + "revlog references a revision not in the index", + ) + })?; + *index_entry.hash() + }; + if node == candidate_node { return Ok(rev); } - if node.is_prefix_of(index_entry.hash()) { + if node.is_prefix_of(&candidate_node) { if found_by_prefix.is_some() { return Err(RevlogError::AmbiguousPrefix); } @@ -913,7 +921,13 @@ .flatten() .collect_vec(); std::fs::write(temp.path().join("foo.i"), contents).unwrap(); - let revlog = Revlog::open(&vfs, "foo.i", None, false).unwrap(); + + let mut idx = nodemap::tests::TestNtIndex::new(); + idx.insert_node(Revision(0), node0).unwrap(); + idx.insert_node(Revision(1), node1).unwrap(); + + let revlog = + Revlog::open_gen(&vfs, "foo.i", None, true, Some(idx.nt)).unwrap(); // accessing the data shows the corruption revlog.get_entry(0.into()).unwrap().data().unwrap_err(); diff -r 752c5a5b73c6 -r 12c308c55e53 rust/hg-core/src/revlog/nodemap.rs --- a/rust/hg-core/src/revlog/nodemap.rs Wed Jan 25 15:34:27 2023 +0100 +++ b/rust/hg-core/src/revlog/nodemap.rs Wed Oct 11 02:02:46 2023 +0200 @@ -693,7 +693,7 @@ } #[cfg(test)] -mod tests { +pub mod tests { use super::NodeMapError::*; use super::*; use crate::revlog::node::{hex_pad_right, Node}; @@ -871,29 +871,36 @@ Ok(()) } - struct TestNtIndex { - index: TestIndex, - nt: NodeTree, + pub struct TestNtIndex { + pub index: TestIndex, + pub nt: NodeTree, } impl TestNtIndex { - fn new() -> Self { + pub fn new() -> Self { TestNtIndex { index: HashMap::new(), nt: NodeTree::default(), } } - fn insert(&mut self, rev: i32, hex: &str) -> Result<(), NodeMapError> { + pub fn insert_node( + &mut self, + rev: Revision, + node: Node, + ) -> Result<(), NodeMapError> { + self.index.insert(rev.into(), node); + self.nt.insert(&self.index, &node, rev)?; + Ok(()) + } + + pub fn insert( + &mut self, + rev: Revision, + hex: &str, + ) -> Result<(), NodeMapError> { let node = pad_node(hex); - let rev: UncheckedRevision = rev.into(); - self.index.insert(rev, node); - self.nt.insert( - &self.index, - &node, - self.index.check_revision(rev).unwrap(), - )?; - Ok(()) + return self.insert_node(rev, node); } fn find_hex( @@ -927,23 +934,23 @@ #[test] fn test_insert_full_mutable() -> Result<(), NodeMapError> { let mut idx = TestNtIndex::new(); - idx.insert(0, "1234")?; + idx.insert(Revision(0), "1234")?; assert_eq!(idx.find_hex("1")?, Some(R!(0))); assert_eq!(idx.find_hex("12")?, Some(R!(0))); // let's trigger a simple split - idx.insert(1, "1a34")?; + idx.insert(Revision(1), "1a34")?; assert_eq!(idx.nt.growable.len(), 1); assert_eq!(idx.find_hex("12")?, Some(R!(0))); assert_eq!(idx.find_hex("1a")?, Some(R!(1))); // reinserting is a no_op - idx.insert(1, "1a34")?; + idx.insert(Revision(1), "1a34")?; assert_eq!(idx.nt.growable.len(), 1); assert_eq!(idx.find_hex("12")?, Some(R!(0))); assert_eq!(idx.find_hex("1a")?, Some(R!(1))); - idx.insert(2, "1a01")?; + idx.insert(Revision(2), "1a01")?; assert_eq!(idx.nt.growable.len(), 2); assert_eq!(idx.find_hex("1a"), Err(NodeMapError::MultipleResults)); assert_eq!(idx.find_hex("12")?, Some(R!(0))); @@ -952,7 +959,7 @@ assert_eq!(idx.find_hex("1a12")?, None); // now let's make it split and create more than one additional block - idx.insert(3, "1a345")?; + idx.insert(Revision(3), "1a345")?; assert_eq!(idx.nt.growable.len(), 4); assert_eq!(idx.find_hex("1a340")?, Some(R!(1))); assert_eq!(idx.find_hex("1a345")?, Some(R!(3))); @@ -966,7 +973,7 @@ #[test] fn test_unique_prefix_len_zero_prefix() { let mut idx = TestNtIndex::new(); - idx.insert(0, "00000abcd").unwrap(); + idx.insert(Revision(0), "00000abcd").unwrap(); assert_eq!(idx.find_hex("000"), Err(NodeMapError::MultipleResults)); // in the nodetree proper, this will be found at the first nybble @@ -976,7 +983,7 @@ assert_eq!(idx.unique_prefix_len_hex("00000ab"), Ok(Some(6))); // same with odd result - idx.insert(1, "00123").unwrap(); + idx.insert(Revision(1), "00123").unwrap(); assert_eq!(idx.unique_prefix_len_hex("001"), Ok(Some(3))); assert_eq!(idx.unique_prefix_len_hex("0012"), Ok(Some(3))); @@ -1012,10 +1019,10 @@ #[test] fn test_insert_partly_immutable() -> Result<(), NodeMapError> { let mut idx = TestNtIndex::new(); - idx.insert(0, "1234")?; - idx.insert(1, "1235")?; - idx.insert(2, "131")?; - idx.insert(3, "cafe")?; + idx.insert(Revision(0), "1234")?; + idx.insert(Revision(1), "1235")?; + idx.insert(Revision(2), "131")?; + idx.insert(Revision(3), "cafe")?; let mut idx = idx.commit(); assert_eq!(idx.find_hex("1234")?, Some(R!(0))); assert_eq!(idx.find_hex("1235")?, Some(R!(1))); @@ -1024,7 +1031,7 @@ // we did not add anything since init from readonly assert_eq!(idx.nt.masked_readonly_blocks(), 0); - idx.insert(4, "123A")?; + idx.insert(Revision(4), "123A")?; assert_eq!(idx.find_hex("1234")?, Some(R!(0))); assert_eq!(idx.find_hex("1235")?, Some(R!(1))); assert_eq!(idx.find_hex("131")?, Some(R!(2))); @@ -1034,7 +1041,7 @@ assert_eq!(idx.nt.masked_readonly_blocks(), 4); eprintln!("{:?}", idx.nt); - idx.insert(5, "c0")?; + idx.insert(Revision(5), "c0")?; assert_eq!(idx.find_hex("cafe")?, Some(R!(3))); assert_eq!(idx.find_hex("c0")?, Some(R!(5))); assert_eq!(idx.find_hex("c1")?, None); @@ -1049,10 +1056,10 @@ #[test] fn test_invalidate_all() -> Result<(), NodeMapError> { let mut idx = TestNtIndex::new(); - idx.insert(0, "1234")?; - idx.insert(1, "1235")?; - idx.insert(2, "131")?; - idx.insert(3, "cafe")?; + idx.insert(Revision(0), "1234")?; + idx.insert(Revision(1), "1235")?; + idx.insert(Revision(2), "131")?; + idx.insert(Revision(3), "cafe")?; let mut idx = idx.commit(); idx.nt.invalidate_all(); @@ -1079,9 +1086,9 @@ #[test] fn test_into_added_bytes() -> Result<(), NodeMapError> { let mut idx = TestNtIndex::new(); - idx.insert(0, "1234")?; + idx.insert(Revision(0), "1234")?; let mut idx = idx.commit(); - idx.insert(4, "cafe")?; + idx.insert(Revision(4), "cafe")?; let (_, bytes) = idx.nt.into_readonly_and_added_bytes(); // only the root block has been changed diff -r 752c5a5b73c6 -r 12c308c55e53 tests/test-contrib-perf.t --- a/tests/test-contrib-perf.t Wed Jan 25 15:34:27 2023 +0100 +++ b/tests/test-contrib-perf.t Wed Oct 11 02:02:46 2023 +0200 @@ -194,7 +194,7 @@ benchmark the full generation of a stream clone perf::stream-locked-section benchmark the initial, repo-locked, section of a stream-clone - perf::tags (no help text available) + perf::tags Benchmark tags retrieval in various situation perf::templating test the rendering time of a given template perf::unbundle diff -r 752c5a5b73c6 -r 12c308c55e53 tests/test-remotefilelog-gc.t --- a/tests/test-remotefilelog-gc.t Wed Jan 25 15:34:27 2023 +0100 +++ b/tests/test-remotefilelog-gc.t Wed Oct 11 02:02:46 2023 +0200 @@ -106,11 +106,6 @@ # Test that warning is displayed when the repo path is malformed $ printf "asdas\0das" >> $CACHEDIR/repos -#if py311 - $ hg gc - finished: removed 0 of 4 files (0.00 GB to 0.00 GB) -#else $ hg gc abort: invalid path asdas\x00da: .*(null|NULL).* (re) [255] -#endif diff -r 752c5a5b73c6 -r 12c308c55e53 tests/test-rhg.t --- a/tests/test-rhg.t Wed Jan 25 15:34:27 2023 +0100 +++ b/tests/test-rhg.t Wed Oct 11 02:02:46 2023 +0200 @@ -27,6 +27,8 @@ Reading and setting configuration $ echo "[ui]" >> $HGRCPATH $ echo "username = user1" >> $HGRCPATH + $ echo "[extensions]" >> $HGRCPATH + $ echo "sparse =" >> $HGRCPATH $ $NO_FALLBACK rhg config ui.username user1 $ echo "[ui]" >> .hg/hgrc @@ -309,6 +311,11 @@ .hg/store/00changelog.i .hg/store/00changelog.n +Rhg status on a sparse repo with nodemap (this specific combination used to crash in 6.5.2) + + $ hg debugsparse -X excluded-dir + $ $NO_FALLBACK rhg status + Specifying revisions by changeset ID $ $NO_FALLBACK rhg files -r c3ae8dec9fad of diff -r 752c5a5b73c6 -r 12c308c55e53 tests/test-share-safe.t --- a/tests/test-share-safe.t Wed Jan 25 15:34:27 2023 +0100 +++ b/tests/test-share-safe.t Wed Oct 11 02:02:46 2023 +0200 @@ -470,7 +470,7 @@ (it is safe to interrupt this process any time before data migration completes) upgrading repository requirements removing temporary repository $TESTTMP/non-share-safe/.hg/upgrade.* (glob) - repository downgraded to not use share safe mode, existing shares will not work and needs to be reshared. + repository downgraded to not use share safe mode, existing shares will not work and need to be reshared. $ hg debugrequirements dotencode diff -r 752c5a5b73c6 -r 12c308c55e53 tests/test-static-http.t --- a/tests/test-static-http.t Wed Jan 25 15:34:27 2023 +0100 +++ b/tests/test-static-http.t Wed Oct 11 02:02:46 2023 +0200 @@ -148,9 +148,17 @@ $ hg paths default = static-http://localhost:$HGPORT/remotempty +test autodetecting static-http: scheme (issue6833) + + $ cd .. + $ hg init actually-static + $ hg clone http://localhost:$HGPORT/actually-static local4 + no changes found + updating to branch default + 0 files updated, 0 files merged, 0 files removed, 0 files unresolved + test with non-repo - $ cd .. $ mkdir notarepo $ hg clone static-http://localhost:$HGPORT/notarepo local3 abort: 'http://localhost:$HGPORT/notarepo' does not appear to be an hg repository @@ -225,6 +233,15 @@ /.hg/store/data/~2ehgsub.i (py37 !) /.hg/store/data/~2ehgsubstate.i (py37 !) /.hg/store/requires + /actually-static/.hg/bookmarks + /actually-static/.hg/bookmarks.current + /actually-static/.hg/dirstate + /actually-static/.hg/requires + /actually-static/.hg/store/00changelog.i + /actually-static/.hg/store/00manifest.i + /actually-static/.hg/store/requires + /actually-static/?cmd=capabilities + /actually-static?cmd=capabilities /notarepo/.hg/00changelog.i /notarepo/.hg/requires /remote-with-names/.hg/bookmarks