changeset 50695:1c31b343e514

match: add `filepath:` pattern to match an exact filepath relative to the root It's useful in certain automated workflows to make sure we recurse in directories whose name conflicts with files in other revisions. In addition it makes it possible to avoid building a potentially costly regex, improving performance when the set of files to match explicitly is large. The benchmark below are run in the following configuration : # data-env-vars.name = mozilla-central-2018-08-01-zstd-sparse-revlog # benchmark.name = files # benchmark.variants.rev = tip # benchmark.variants.files = all-list-filepath-sorted # bin-env-vars.hg.flavor = no-rust It also includes timings using the re2 engine (through the `google-re2` module) to show how much can be saved by just using a better regexp engine. Pattern time (seconds) time using re2 ----------------------------------------------------------- just "." 0.4 0.4 list of "filepath:…" 1.3 1.3 list of "path:…" 25.7 3.9 list of patterns 29.7 10.4 As you can see, Without re2, using "filepath:" instead of "path:" is a huge win. With re2, it is still about three times faster to not have to build the regex.
author Raphaël Gomès <rgomes@octobus.net>
date Mon, 12 Jun 2023 16:51:08 +0200
parents a41eeb877d07
children eb01d3a65ad8
files mercurial/helptext/patterns.txt mercurial/match.py rust/hg-core/src/filepatterns.rs rust/hg-core/src/matchers.rs tests/test-match.py tests/test-walk.t
diffstat 6 files changed, 168 insertions(+), 15 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial/helptext/patterns.txt	Sun Jun 18 00:09:39 2023 +0200
+++ b/mercurial/helptext/patterns.txt	Mon Jun 12 16:51:08 2023 +0200
@@ -18,7 +18,8 @@
 current repository root, and when the path points to a directory, it is matched
 recursively. To match all files in a directory non-recursively (not including
 any files in subdirectories), ``rootfilesin:`` can be used, specifying an
-absolute path (relative to the repository root).
+absolute path (relative to the repository root). To match a single file exactly,
+relative to the repository root, you can use ``filepath:``.
 
 To use an extended glob, start a name with ``glob:``. Globs are rooted
 at the current directory; a glob such as ``*.c`` will only match files
@@ -50,11 +51,15 @@
 
 Plain examples::
 
-  path:foo/bar        a name bar in a directory named foo in the root
-                      of the repository
-  path:path:name      a file or directory named "path:name"
-  rootfilesin:foo/bar the files in a directory called foo/bar, but not any files
-                      in its subdirectories and not a file bar in directory foo
+  path:foo/bar                  a name bar in a directory named foo in the root
+                                of the repository
+  path:some/path                a file or directory named "some/path"
+  filepath:some/path/to/a/file  exactly a single file named
+                                "some/path/to/a/file", relative to the root
+                                of the repository
+  rootfilesin:foo/bar           the files in a directory called foo/bar, but
+                                not any files in its subdirectories and not
+                                a file bar in directory foo
 
 Glob examples::
 
--- a/mercurial/match.py	Sun Jun 18 00:09:39 2023 +0200
+++ b/mercurial/match.py	Mon Jun 12 16:51:08 2023 +0200
@@ -30,6 +30,7 @@
     b're',
     b'glob',
     b'path',
+    b'filepath',
     b'relglob',
     b'relpath',
     b'relre',
@@ -181,6 +182,8 @@
     're:<regexp>' - a regular expression
     'path:<path>' - a path relative to repository root, which is matched
                     recursively
+    'filepath:<path>' - an exact path to a single file, relative to the
+                        repository root
     'rootfilesin:<path>' - a path relative to repository root, which is
                     matched non-recursively (will not match subdirectories)
     'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
@@ -334,10 +337,18 @@
     """Convert 'kind:pat' from the patterns list to tuples with kind and
     normalized and rooted patterns and with listfiles expanded."""
     kindpats = []
+    kinds_to_normalize = (
+        b'relglob',
+        b'path',
+        b'filepath',
+        b'rootfilesin',
+        b'rootglob',
+    )
+
     for kind, pat in [_patsplit(p, default) for p in patterns]:
         if kind in cwdrelativepatternkinds:
             pat = pathutil.canonpath(root, cwd, pat, auditor=auditor)
-        elif kind in (b'relglob', b'path', b'rootfilesin', b'rootglob'):
+        elif kind in kinds_to_normalize:
             pat = util.normpath(pat)
         elif kind in (b'listfile', b'listfile0'):
             try:
@@ -1340,6 +1351,10 @@
         return b''
     if kind == b're':
         return pat
+    if kind == b'filepath':
+        raise error.ProgrammingError(
+            "'filepath:' patterns should not be converted to a regex"
+        )
     if kind in (b'path', b'relpath'):
         if pat == b'.':
             return b''
@@ -1444,7 +1459,14 @@
     """
     try:
         allgroups = []
-        regexps = [_regex(k, p, globsuffix) for (k, p, s) in kindpats]
+        regexps = []
+        exact = set()
+        for (kind, pattern, _source) in kindpats:
+            if kind == b'filepath':
+                exact.add(pattern)
+                continue
+            regexps.append(_regex(kind, pattern, globsuffix))
+
         fullregexp = _joinregexes(regexps)
 
         startidx = 0
@@ -1469,9 +1491,20 @@
             allgroups.append(_joinregexes(group))
             allmatchers = [_rematcher(g) for g in allgroups]
             func = lambda s: any(m(s) for m in allmatchers)
-        return fullregexp, func
+
+        actualfunc = func
+        if exact:
+            # An empty regex will always match, so only call the regex if
+            # there were any actual patterns to match.
+            if not regexps:
+                actualfunc = lambda s: s in exact
+            else:
+                actualfunc = lambda s: s in exact or func(s)
+        return fullregexp, actualfunc
     except re.error:
         for k, p, s in kindpats:
+            if k == b'filepath':
+                continue
             try:
                 _rematcher(_regex(k, p, globsuffix))
             except re.error:
@@ -1502,7 +1535,7 @@
                     break
                 root.append(p)
             r.append(b'/'.join(root))
-        elif kind in (b'relpath', b'path'):
+        elif kind in (b'relpath', b'path', b'filepath'):
             if pat == b'.':
                 pat = b''
             r.append(pat)
--- a/rust/hg-core/src/filepatterns.rs	Sun Jun 18 00:09:39 2023 +0200
+++ b/rust/hg-core/src/filepatterns.rs	Mon Jun 12 16:51:08 2023 +0200
@@ -50,6 +50,8 @@
     Glob,
     /// a path relative to repository root, which is matched recursively
     Path,
+    /// a single exact path relative to repository root
+    FilePath,
     /// A path relative to cwd
     RelPath,
     /// an unrooted glob (*.rs matches Rust files in all dirs)
@@ -157,6 +159,7 @@
     match kind {
         b"re:" => Ok(PatternSyntax::Regexp),
         b"path:" => Ok(PatternSyntax::Path),
+        b"filepath:" => Ok(PatternSyntax::FilePath),
         b"relpath:" => Ok(PatternSyntax::RelPath),
         b"rootfilesin:" => Ok(PatternSyntax::RootFiles),
         b"relglob:" => Ok(PatternSyntax::RelGlob),
@@ -252,7 +255,8 @@
         }
         PatternSyntax::Include
         | PatternSyntax::SubInclude
-        | PatternSyntax::ExpandedSubInclude(_) => unreachable!(),
+        | PatternSyntax::ExpandedSubInclude(_)
+        | PatternSyntax::FilePath => unreachable!(),
     }
 }
 
@@ -319,9 +323,9 @@
         }
         _ => pattern.to_owned(),
     };
-    if *syntax == PatternSyntax::RootGlob
-        && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b))
-    {
+    let is_simple_rootglob = *syntax == PatternSyntax::RootGlob
+        && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b));
+    if is_simple_rootglob || syntax == &PatternSyntax::FilePath {
         Ok(None)
     } else {
         let mut entry = entry.clone();
--- a/rust/hg-core/src/matchers.rs	Sun Jun 18 00:09:39 2023 +0200
+++ b/rust/hg-core/src/matchers.rs	Mon Jun 12 16:51:08 2023 +0200
@@ -708,7 +708,9 @@
                 }
                 roots.push(root);
             }
-            PatternSyntax::Path | PatternSyntax::RelPath => {
+            PatternSyntax::Path
+            | PatternSyntax::RelPath
+            | PatternSyntax::FilePath => {
                 let pat = HgPath::new(if pattern == b"." {
                     &[] as &[u8]
                 } else {
@@ -1223,6 +1225,40 @@
             VisitChildrenSet::This
         );
 
+        // VisitchildrensetFilePath
+        let matcher = IncludeMatcher::new(vec![IgnorePattern::new(
+            PatternSyntax::FilePath,
+            b"dir/z",
+            Path::new(""),
+        )])
+        .unwrap();
+
+        let mut set = HashSet::new();
+        set.insert(HgPathBuf::from_bytes(b"dir"));
+        assert_eq!(
+            matcher.visit_children_set(HgPath::new(b"")),
+            VisitChildrenSet::Set(set)
+        );
+        assert_eq!(
+            matcher.visit_children_set(HgPath::new(b"folder")),
+            VisitChildrenSet::Empty
+        );
+        let mut set = HashSet::new();
+        set.insert(HgPathBuf::from_bytes(b"z"));
+        assert_eq!(
+            matcher.visit_children_set(HgPath::new(b"dir")),
+            VisitChildrenSet::Set(set)
+        );
+        // OPT: these should probably be set().
+        assert_eq!(
+            matcher.visit_children_set(HgPath::new(b"dir/subdir")),
+            VisitChildrenSet::Empty
+        );
+        assert_eq!(
+            matcher.visit_children_set(HgPath::new(b"dir/subdir/x")),
+            VisitChildrenSet::Empty
+        );
+
         // Test multiple patterns
         let matcher = IncludeMatcher::new(vec![
             IgnorePattern::new(PatternSyntax::RelPath, b"foo", Path::new("")),
--- a/tests/test-match.py	Sun Jun 18 00:09:39 2023 +0200
+++ b/tests/test-match.py	Mon Jun 12 16:51:08 2023 +0200
@@ -140,6 +140,28 @@
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
 
+    def testVisitdirFilepath(self):
+        m = matchmod.match(
+            util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z']
+        )
+        assert isinstance(m, matchmod.patternmatcher)
+        self.assertTrue(m.visitdir(b''))
+        self.assertTrue(m.visitdir(b'dir'))
+        self.assertFalse(m.visitdir(b'folder'))
+        self.assertFalse(m.visitdir(b'dir/subdir'))
+        self.assertFalse(m.visitdir(b'dir/subdir/x'))
+
+    def testVisitchildrensetFilepath(self):
+        m = matchmod.match(
+            util.localpath(b'/repo'), b'', patterns=[b'filepath:dir/z']
+        )
+        assert isinstance(m, matchmod.patternmatcher)
+        self.assertEqual(m.visitchildrenset(b''), b'this')
+        self.assertEqual(m.visitchildrenset(b'folder'), set())
+        self.assertEqual(m.visitchildrenset(b'dir'), b'this')
+        self.assertEqual(m.visitchildrenset(b'dir/subdir'), set())
+        self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
+
 
 class IncludeMatcherTests(unittest.TestCase):
     def testVisitdirPrefix(self):
@@ -212,6 +234,28 @@
         self.assertEqual(m.visitchildrenset(b'dir/subdir'), b'this')
         self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), b'this')
 
+    def testVisitdirFilepath(self):
+        m = matchmod.match(
+            util.localpath(b'/repo'), b'', include=[b'filepath:dir/z']
+        )
+        assert isinstance(m, matchmod.includematcher)
+        self.assertTrue(m.visitdir(b''))
+        self.assertTrue(m.visitdir(b'dir'))
+        self.assertFalse(m.visitdir(b'folder'))
+        self.assertFalse(m.visitdir(b'dir/subdir'))
+        self.assertFalse(m.visitdir(b'dir/subdir/x'))
+
+    def testVisitchildrensetFilepath(self):
+        m = matchmod.match(
+            util.localpath(b'/repo'), b'', include=[b'filepath:dir/z']
+        )
+        assert isinstance(m, matchmod.includematcher)
+        self.assertEqual(m.visitchildrenset(b''), {b'dir'})
+        self.assertEqual(m.visitchildrenset(b'folder'), set())
+        self.assertEqual(m.visitchildrenset(b'dir'), {b'z'})
+        self.assertEqual(m.visitchildrenset(b'dir/subdir'), set())
+        self.assertEqual(m.visitchildrenset(b'dir/subdir/x'), set())
+
 
 class ExactMatcherTests(unittest.TestCase):
     def testVisitdir(self):
--- a/tests/test-walk.t	Sun Jun 18 00:09:39 2023 +0200
+++ b/tests/test-walk.t	Mon Jun 12 16:51:08 2023 +0200
@@ -61,6 +61,37 @@
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
 
+Test 'filepath:' pattern
+
+  $ hg debugwalk -v -I 'filepath:mammals/Procyonidae/cacomistle'
+  * matcher:
+  <includematcher includes=''>
+  f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
+
+  $ hg debugwalk -v -I 'filepath:mammals/Procyonidae'
+  * matcher:
+  <includematcher includes=''>
+
+  $ hg debugwalk -v -X 'filepath:beans/borlotti'
+  * matcher:
+  <differencematcher
+    m1=<alwaysmatcher>,
+    m2=<includematcher includes=''>>
+  f  beans/black                     beans/black
+  f  beans/kidney                    beans/kidney
+  f  beans/navy                      beans/navy
+  f  beans/pinto                     beans/pinto
+  f  beans/turtle                    beans/turtle
+  f  fennel                          fennel
+  f  fenugreek                       fenugreek
+  f  fiddlehead                      fiddlehead
+  f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
+  f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
+  f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
+  f  mammals/skunk                   mammals/skunk
+
+Test relative paths
+
   $ cd mammals
   $ hg debugwalk -v
   * matcher: