Mercurial > hg
changeset 50695:1c31b343e514
match: add `filepath:` pattern to match an exact filepath relative to the root
It's useful in certain automated workflows to make sure we recurse in
directories whose name conflicts with files in other revisions.
In addition it makes it possible to avoid building a potentially costly regex,
improving performance when the set of files to match explicitly is large.
The benchmark below are run in the following configuration :
# data-env-vars.name = mozilla-central-2018-08-01-zstd-sparse-revlog
# benchmark.name = files
# benchmark.variants.rev = tip
# benchmark.variants.files = all-list-filepath-sorted
# bin-env-vars.hg.flavor = no-rust
It also includes timings using the re2 engine (through the `google-re2` module)
to show how much can be saved by just using a better regexp engine.
Pattern time (seconds) time using re2
-----------------------------------------------------------
just "." 0.4 0.4
list of "filepath:…" 1.3 1.3
list of "path:…" 25.7 3.9
list of patterns 29.7 10.4
As you can see, Without re2, using "filepath:" instead of "path:" is a huge
win. With re2, it is still about three times faster to not have to build the
regex.
| author | Raphaël Gomès <rgomes@octobus.net> |
|---|---|
| date | Mon, 12 Jun 2023 16:51:08 +0200 |
| parents | a41eeb877d07 |
| children | eb01d3a65ad8 |
| files | mercurial/helptext/patterns.txt mercurial/match.py rust/hg-core/src/filepatterns.rs rust/hg-core/src/matchers.rs tests/test-match.py tests/test-walk.t |
| diffstat | 6 files changed, 168 insertions(+), 15 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/helptext/patterns.txt Sun Jun 18 00:09:39 2023 +0200 +++ b/mercurial/helptext/patterns.txt Mon Jun 12 16:51:08 2023 +0200 @@ -18,7 +18,8 @@ current repository root, and when the path points to a directory, it is matched recursively. To match all files in a directory non-recursively (not including any files in subdirectories), ``rootfilesin:`` can be used, specifying an -absolute path (relative to the repository root). +absolute path (relative to the repository root). To match a single file exactly, +relative to the repository root, you can use ``filepath:``. To use an extended glob, start a name with ``glob:``. Globs are rooted at the current directory; a glob such as ``*.c`` will only match files @@ -50,11 +51,15 @@ Plain examples:: - path:foo/bar a name bar in a directory named foo in the root - of the repository - path:path:name a file or directory named "path:name" - rootfilesin:foo/bar the files in a directory called foo/bar, but not any files - in its subdirectories and not a file bar in directory foo + path:foo/bar a name bar in a directory named foo in the root + of the repository + path:some/path a file or directory named "some/path" + filepath:some/path/to/a/file exactly a single file named + "some/path/to/a/file", relative to the root + of the repository + rootfilesin:foo/bar the files in a directory called foo/bar, but + not any files in its subdirectories and not + a file bar in directory foo Glob examples::
--- a/mercurial/match.py Sun Jun 18 00:09:39 2023 +0200 +++ b/mercurial/match.py Mon Jun 12 16:51:08 2023 +0200 @@ -30,6 +30,7 @@ b're', b'glob', b'path', + b'filepath', b'relglob', b'relpath', b'relre', @@ -181,6 +182,8 @@ 're:<regexp>' - a regular expression 'path:<path>' - a path relative to repository root, which is matched recursively + 'filepath:<path>' - an exact path to a single file, relative to the + repository root 'rootfilesin:<path>' - a path relative to repository root, which is matched non-recursively (will not match subdirectories) 'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs) @@ -334,10 +337,18 @@ """Convert 'kind:pat' from the patterns list to tuples with kind and normalized and rooted patterns and with listfiles expanded.""" kindpats = [] + kinds_to_normalize = ( + b'relglob', + b'path', + b'filepath', + b'rootfilesin', + b'rootglob', + ) + for kind, pat in [_patsplit(p, default) for p in patterns]: if kind in cwdrelativepatternkinds: pat = pathutil.canonpath(root, cwd, pat, auditor=auditor) - elif kind in (b'relglob', b'path', b'rootfilesin', b'rootglob'): + elif kind in kinds_to_normalize: pat = util.normpath(pat) elif kind in (b'listfile', b'listfile0'): try: @@ -1340,6 +1351,10 @@ return b'' if kind == b're': return pat + if kind == b'filepath': + raise error.ProgrammingError( + "'filepath:' patterns should not be converted to a regex" + ) if kind in (b'path', b'relpath'): if pat == b'.': return b'' @@ -1444,7 +1459,14 @@ """ try: allgroups = [] - regexps = [_regex(k, p, globsuffix) for (k, p, s) in kindpats] + regexps = [] + exact = set() + for (kind, pattern, _source) in kindpats: + if kind == b'filepath': + exact.add(pattern) + continue + regexps.append(_regex(kind, pattern, globsuffix)) + fullregexp = _joinregexes(regexps) startidx = 0 @@ -1469,9 +1491,20 @@ allgroups.append(_joinregexes(group)) allmatchers = [_rematcher(g) for g in allgroups] func = lambda s: any(m(s) for m in allmatchers) - return fullregexp, func + + actualfunc = func + if exact: + # An empty regex will always match, so only call the regex if + # there were any actual patterns to match. + if not regexps: + actualfunc = lambda s: s in exact + else: + actualfunc = lambda s: s in exact or func(s) + return fullregexp, actualfunc except re.error: for k, p, s in kindpats: + if k == b'filepath': + continue try: _rematcher(_regex(k, p, globsuffix)) except re.error: @@ -1502,7 +1535,7 @@ break root.append(p) r.append(b'/'.join(root)) - elif kind in (b'relpath', b'path'): + elif kind in (b'relpath', b'path', b'filepath'): if pat == b'.': pat = b'' r.append(pat)
--- a/rust/hg-core/src/filepatterns.rs Sun Jun 18 00:09:39 2023 +0200 +++ b/rust/hg-core/src/filepatterns.rs Mon Jun 12 16:51:08 2023 +0200 @@ -50,6 +50,8 @@ Glob, /// a path relative to repository root, which is matched recursively Path, + /// a single exact path relative to repository root + FilePath, /// A path relative to cwd RelPath, /// an unrooted glob (*.rs matches Rust files in all dirs) @@ -157,6 +159,7 @@ match kind { b"re:" => Ok(PatternSyntax::Regexp), b"path:" => Ok(PatternSyntax::Path), + b"filepath:" => Ok(PatternSyntax::FilePath), b"relpath:" => Ok(PatternSyntax::RelPath), b"rootfilesin:" => Ok(PatternSyntax::RootFiles), b"relglob:" => Ok(PatternSyntax::RelGlob), @@ -252,7 +255,8 @@ } PatternSyntax::Include | PatternSyntax::SubInclude - | PatternSyntax::ExpandedSubInclude(_) => unreachable!(), + | PatternSyntax::ExpandedSubInclude(_) + | PatternSyntax::FilePath => unreachable!(), } } @@ -319,9 +323,9 @@ } _ => pattern.to_owned(), }; - if *syntax == PatternSyntax::RootGlob - && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b)) - { + let is_simple_rootglob = *syntax == PatternSyntax::RootGlob + && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b)); + if is_simple_rootglob || syntax == &PatternSyntax::FilePath { Ok(None) } else { let mut entry = entry.clone();
--- a/rust/hg-core/src/matchers.rs Sun Jun 18 00:09:39 2023 +0200 +++ b/rust/hg-core/src/matchers.rs Mon Jun 12 16:51:08 2023 +0200 @@ -708,7 +708,9 @@ } roots.push(root); } - PatternSyntax::Path | PatternSyntax::RelPath => { + PatternSyntax::Path + | PatternSyntax::RelPath + | PatternSyntax::FilePath => { let pat = HgPath::new(if pattern == b"." { &[] as &[u8] } else { @@ -1223,6 +1225,40 @@ VisitChildrenSet::This ); + // VisitchildrensetFilePath + let matcher = IncludeMatcher::new(vec![IgnorePattern::new( + PatternSyntax::FilePath, + b"dir/z", + Path::new(""), + )]) + .unwrap(); + + let mut set = HashSet::new(); + set.insert(HgPathBuf::from_bytes(b"dir")); + assert_eq!( + matcher.visit_children_set(HgPath::new(b"")), + VisitChildrenSet::Set(set) + ); + assert_eq!( + matcher.visit_children_set(HgPath::new(b"folder")), + VisitChildrenSet::Empty + ); + let mut set = HashSet::new(); + set.insert(HgPathBuf::from_bytes(b"z")); + assert_eq!( + matcher.visit_children_set(HgPath::new(b"dir")), + VisitChildrenSet::Set(set) + ); + // OPT: these should probably be set(). + assert_eq!( + matcher.visit_children_set(HgPath::new(b"dir/subdir")), + VisitChildrenSet::Empty + ); + assert_eq!( + matcher.visit_children_set(HgPath::new(b"dir/subdir/x")), + VisitChildrenSet::Empty + ); + // Test multiple patterns let matcher = IncludeMatcher::new(vec![ IgnorePattern::new(PatternSyntax::RelPath, b"foo", Path::new("")),
--- a/tests/test-match.py Sun Jun 18 00:09:39 2023 +0200 +++ b/tests/test-match.py Mon Jun 12 16:51:08 2023 +0200 @@ -140,6 +140,28 @@ self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this') self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this') + def testVisitdirFilepath(self): + m = matchmod.match( + util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z'] + ) + assert isinstance(m, matchmod.patternmatcher) + self.assertTrue(m.visitdir(b'')) + self.assertTrue(m.visitdir(b'dir')) + self.assertFalse(m.visitdir(b'folder')) + self.assertFalse(m.visitdir(b'dir/subdir')) + self.assertFalse(m.visitdir(b'dir/subdir/x')) + + def testVisitchildrensetFilepath(self): + m = matchmod.match( + util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z'] + ) + assert isinstance(m, matchmod.patternmatcher) + self.assertEqual(m.visitchildrenset(b''), b'this') + self.assertEqual(m.visitchildrenset(b'folder'), set()) + self.assertEqual(m.visitchildrenset(b'dir'), b'this') + self.assertEqual(m.visitchildrenset(b'dir/subdir'), set()) + self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set()) + class IncludeMatcherTests(unittest.TestCase): def testVisitdirPrefix(self): @@ -212,6 +234,28 @@ self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this') self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this') + def testVisitdirFilepath(self): + m = matchmod.match( + util.localpath(b'/repo'), b'', include=[b'filepath:dir/z'] + ) + assert isinstance(m, matchmod.includematcher) + self.assertTrue(m.visitdir(b'')) + self.assertTrue(m.visitdir(b'dir')) + self.assertFalse(m.visitdir(b'folder')) + self.assertFalse(m.visitdir(b'dir/subdir')) + self.assertFalse(m.visitdir(b'dir/subdir/x')) + + def testVisitchildrensetFilepath(self): + m = matchmod.match( + util.localpath(b'/repo'), b'', include=[b'filepath:dir/z'] + ) + assert isinstance(m, matchmod.includematcher) + self.assertEqual(m.visitchildrenset(b''), {b'dir'}) + self.assertEqual(m.visitchildrenset(b'folder'), set()) + self.assertEqual(m.visitchildrenset(b'dir'), {b'z'}) + self.assertEqual(m.visitchildrenset(b'dir/subdir'), set()) + self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set()) + class ExactMatcherTests(unittest.TestCase): def testVisitdir(self):
--- a/tests/test-walk.t Sun Jun 18 00:09:39 2023 +0200 +++ b/tests/test-walk.t Mon Jun 12 16:51:08 2023 +0200 @@ -61,6 +61,37 @@ f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon f mammals/skunk mammals/skunk +Test 'filepath:' pattern + + $ hg debugwalk -v -I 'filepath:mammals/Procyonidae/cacomistle' + * matcher: + <includematcher includes=''> + f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle + + $ hg debugwalk -v -I 'filepath:mammals/Procyonidae' + * matcher: + <includematcher includes=''> + + $ hg debugwalk -v -X 'filepath:beans/borlotti' + * matcher: + <differencematcher + m1=<alwaysmatcher>, + m2=<includematcher includes=''>> + f beans/black beans/black + f beans/kidney beans/kidney + f beans/navy beans/navy + f beans/pinto beans/pinto + f beans/turtle beans/turtle + f fennel fennel + f fenugreek fenugreek + f fiddlehead fiddlehead + f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle + f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi + f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon + f mammals/skunk mammals/skunk + +Test relative paths + $ cd mammals $ hg debugwalk -v * matcher: