Mercurial: rust/hg-core/src/dirstate_tree/status.rs comparison

comparison rust/hg-core/src/dirstate_tree/status.rs @ 49568:da48f170d203

rust-status: query fs traversal metadata lazily Currently, any time the status algorithm needs to read a directory from the filesystem (because the stat-only optimization is not available), it also stats each directory entry eagerly. Stat'ing the entries is only needed in a few cases (like when checking the mtime of a directory for caching): this patch creates a wrapper struct `DirEntry` that only stats the directory entry it represents when needed. Excerpt of an `strace` before this change on Mozilla Central: ``` openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 newfstatat(3, "", {st_mode=S_IFDIR|0755, st_size=3540, ...}, AT_EMPTY_PATH) = 0 getdents64(3, 0x55dc970bd440 /* 139 entries */, 32768) = 5072 statx(3, ".hg", AT_STATX_SYNC_AS_STAT|AT_SYMLINK_NOFOLLOW, STATX_ALL, {stx_mask=STATX_ALL|STATX_MNT_ID, stx_attributes=0, stx_mode=S_IFDIR|0755, stx_size=772, ...}) = 0 [... 135 other successful `statx` calls] getdents64(3, 0x55dc970bd440 /* 0 entries */, 32768) = 0 close(3) = 0 ``` After this change: ``` openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 newfstatat(3, "", {st_mode=S_IFDIR|0755, st_size=3540, ...}, AT_EMPTY_PATH) = 0 getdents64(3, 0x561567c10190 /* 139 entries */, 32768) = 5072 getdents64(3, 0x561567c10190 /* 0 entries */, 32768) = 0 close(3) = 0 ```

author	Raphaël Gomès <rgomes@octobus.net>
date	Wed, 19 Oct 2022 12:38:06 +0200
parents	6b32d39e9a67
children	18282cf18aa2

comparison

equal deleted inserted replaced

-:6b32d39e9a67
+:da48f170d203
 use crate::utils::files::get_bytes_from_os_string;
 use crate::utils::files::get_path_from_bytes;
 use crate::utils::hg_path::HgPath;
 use crate::BadMatch;
 use crate::DirstateStatus;
-use crate::HgPathBuf;
 use crate::HgPathCow;
 use crate::PatternFileWarning;
 use crate::StatusError;
 use crate::StatusOptions;
 use micro_timer::timed;
 use once_cell::sync::OnceCell;
 use rayon::prelude::*;
 use sha1::{Digest, Sha1};
 use std::borrow::Cow;
+use std::convert::TryFrom;
+use std::convert::TryInto;
 use std::io;
 use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Mutex;
 use std::time::SystemTime;
 };
 let is_at_repo_root = true;
 let hg_path = &BorrowedPath::OnDisk(HgPath::new(""));
 let has_ignored_ancestor = HasIgnoredAncestor::create(None, hg_path);
 let root_cached_mtime = None;
-let root_dir_metadata = None;
 // If the path we have for the repository root is a symlink, do follow it.
 // (As opposed to symlinks within the working directory which are not
 // followed, using `std::fs::symlink_metadata`.)
 common.traverse_fs_directory_and_dirstate(
 &has_ignored_ancestor,
 dmap.root.as_ref(),
 hg_path,
-&root_dir,
+&DirEntry {
-root_dir_metadata,
+hg_path: Cow::Borrowed(HgPath::new(b"")),
+fs_path: Cow::Borrowed(&root_dir),
+symlink_metadata: None,
+file_type: FakeFileType::Directory,
+},
 root_cached_mtime,
 is_at_repo_root,
 )?;
 let mut outcome = common.outcome.into_inner().unwrap();
 let new_cachable = common.new_cachable_directories.into_inner().unwrap();
 /// If this returns true, we can get accurate results by only using
 /// `symlink_metadata` for child nodes that exist in the dirstate and don’t
 /// need to call `read_dir`.
 fn can_skip_fs_readdir(
 &self,
-directory_metadata: Option<&std::fs::Metadata>,
+directory_entry: &DirEntry,
 cached_directory_mtime: Option<TruncatedTimestamp>,
 ) -> bool {
 if !self.options.list_unknown && !self.options.list_ignored {
 // All states that we care about listing have corresponding
 // dirstate entries.
 {
 if let Some(cached_mtime) = cached_directory_mtime {
 // The dirstate contains a cached mtime for this directory, set
 // by a previous run of the `status` algorithm which found this
 // directory eligible for `read_dir` caching.
-if let Some(meta) = directory_metadata {
+if let Ok(meta) = directory_entry.symlink_metadata() {
 if cached_mtime
-.likely_equal_to_mtime_of(meta)
+.likely_equal_to_mtime_of(&meta)
 .unwrap_or(false)
 {
 // The mtime of that directory has not changed
 // since then, which means that the results of
 // `read_dir` should also be unchanged.
 fn traverse_fs_directory_and_dirstate<'ancestor>(
 &self,
 has_ignored_ancestor: &'ancestor HasIgnoredAncestor<'ancestor>,
 dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
 directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
-directory_fs_path: &Path,
+directory_entry: &DirEntry,
-directory_metadata: Option<&std::fs::Metadata>,
 cached_directory_mtime: Option<TruncatedTimestamp>,
 is_at_repo_root: bool,
 ) -> Result<bool, DirstateV2ParseError> {
-if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime)
+if self.can_skip_fs_readdir(directory_entry, cached_directory_mtime) {
-{
 dirstate_nodes
 .par_iter()
 .map(|dirstate_node| {
-let fs_path = directory_fs_path.join(get_path_from_bytes(
+let fs_path = &directory_entry.fs_path;
+let fs_path = fs_path.join(get_path_from_bytes(
 dirstate_node.base_name(self.dmap.on_disk)?.as_bytes(),
 ));
 match std::fs::symlink_metadata(&fs_path) {
-Ok(fs_metadata) => self.traverse_fs_and_dirstate(
+Ok(fs_metadata) => {
-&fs_path,
+let file_type =
-&fs_metadata,
+match fs_metadata.file_type().try_into() {
-dirstate_node,
+Ok(file_type) => file_type,
-has_ignored_ancestor,
+Err(_) => return Ok(()),
-),
+};
+let entry = DirEntry {
+hg_path: Cow::Borrowed(
+dirstate_node
+.full_path(&self.dmap.on_disk)?,
+),
+fs_path: Cow::Borrowed(&fs_path),
+symlink_metadata: Some(fs_metadata),
+file_type,
+};
+self.traverse_fs_and_dirstate(
+&entry,
+dirstate_node,
+has_ignored_ancestor,
+)
+}
 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
 self.traverse_dirstate_only(dirstate_node)
 }
 Err(error) => {
 let hg_path =
 return Ok(children_all_have_dirstate_node_or_are_ignored);
 }
 let mut fs_entries = if let Ok(entries) = self.read_dir(
 directory_hg_path,
-directory_fs_path,
+&directory_entry.fs_path,
 is_at_repo_root,
 ) {
 entries
 } else {
 // Treat an unreadable directory (typically because of insufficient
 use itertools::EitherOrBoth::*;
 let has_dirstate_node_or_is_ignored;
 match pair {
 Both(dirstate_node, fs_entry) => {
 self.traverse_fs_and_dirstate(
-&fs_entry.fs_path,
+&fs_entry,
-&fs_entry.metadata,
 dirstate_node,
 has_ignored_ancestor,
 )?;
 has_dirstate_node_or_is_ignored = true
 }
 .try_reduce(|| true, |a, b| Ok(a && b))
 }
 fn traverse_fs_and_dirstate<'ancestor>(
 &self,
-fs_path: &Path,
+fs_entry: &DirEntry,
-fs_metadata: &std::fs::Metadata,
 dirstate_node: NodeRef<'tree, 'on_disk>,
 has_ignored_ancestor: &'ancestor HasIgnoredAncestor<'ancestor>,
 ) -> Result<(), DirstateV2ParseError> {
 self.check_for_outdated_directory_cache(&dirstate_node)?;
 let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
-let file_type = fs_metadata.file_type();
+let file_or_symlink = fs_entry.is_file() || fs_entry.is_symlink();
-let file_or_symlink = file_type.is_file() || file_type.is_symlink();
 if !file_or_symlink {
 // If we previously had a file here, it was removed (with
 // `hg rm` or similar) or deleted before it could be
 // replaced by a directory or something else.
 self.mark_removed_or_deleted_if_file(&dirstate_node)?;
 }
-if file_type.is_dir() {
+if fs_entry.is_dir() {
 if self.options.collect_traversed_dirs {
 self.outcome
 .lock()
 .unwrap()
 .traversed
 let children_all_have_dirstate_node_or_are_ignored = self
 .traverse_fs_directory_and_dirstate(
 &is_ignored,
 dirstate_node.children(self.dmap.on_disk)?,
 hg_path,
-fs_path,
+fs_entry,
-Some(fs_metadata),
 dirstate_node.cached_directory_mtime()?,
 is_at_repo_root,
 )?;
 self.maybe_save_directory_mtime(
 children_all_have_dirstate_node_or_are_ignored,
-fs_metadata,
+fs_entry,
 dirstate_node,
 )?
 } else {
 if file_or_symlink && self.matcher.matches(&hg_path) {
 if let Some(entry) = dirstate_node.entry()? {
 } else if entry.removed() {
 self.push_outcome(Outcome::Removed, &dirstate_node)?;
 } else if entry.modified() {
 self.push_outcome(Outcome::Modified, &dirstate_node)?;
 } else {
-self.handle_normal_file(&dirstate_node, fs_metadata)?;
+self.handle_normal_file(&dirstate_node, fs_entry)?;
 }
 } else {
 // `node.entry.is_none()` indicates a "directory"
 // node, but the filesystem has a file
 self.mark_unknown_or_ignored(
 }
 fn maybe_save_directory_mtime(
 &self,
 children_all_have_dirstate_node_or_are_ignored: bool,
-directory_metadata: &std::fs::Metadata,
+directory_entry: &DirEntry,
 dirstate_node: NodeRef<'tree, 'on_disk>,
 ) -> Result<(), DirstateV2ParseError> {
 if !children_all_have_dirstate_node_or_are_ignored {
 return Ok(());
 }
 // has nanosecond precision, the times reported for a
 // directory’s (or file’s) modified time may have lower
 // resolution based on the filesystem (for example ext3
 // only stores integer seconds), kernel (see
 // https://stackoverflow.com/a/14393315/1162888), etc.
+let metadata = match directory_entry.symlink_metadata() {
+Ok(meta) => meta,
+Err(_) => return Ok(()),
+};
 let directory_mtime = if let Ok(option) =
-TruncatedTimestamp::for_reliable_mtime_of(
+TruncatedTimestamp::for_reliable_mtime_of(&metadata, status_start)
-directory_metadata,
+{
-status_start,
-) {
 if let Some(directory_mtime) = option {
 directory_mtime
 } else {
 // The directory was modified too recently,
 // don’t cache its `read_dir` results.
 /// A file that is clean in the dirstate was found in the filesystem
 fn handle_normal_file(
 &self,
 dirstate_node: &NodeRef<'tree, 'on_disk>,
-fs_metadata: &std::fs::Metadata,
+fs_entry: &DirEntry,
 ) -> Result<(), DirstateV2ParseError> {
 // Keep the low 31 bits
 fn truncate_u64(value: u64) -> i32 {
 (value & 0x7FFF_FFFF) as i32
 }
+let fs_metadata = match fs_entry.symlink_metadata() {
+Ok(meta) => meta,
+Err(_) => return Ok(()),
+};
 let entry = dirstate_node
 .entry()?
 .expect("handle_normal_file called with entry-less node");
 let mode_changed =
-|| self.options.check_exec && entry.mode_changed(fs_metadata);
+|| self.options.check_exec && entry.mode_changed(&fs_metadata);
 let size = entry.size();
 let size_changed = size != truncate_u64(fs_metadata.len());
 if size >= 0 && size_changed && fs_metadata.file_type().is_symlink() {
 // issue6456: Size returned may be longer due to encryption
 // on EXT-4 fscrypt. TODO maybe only do it on EXT4?
 {
 self.push_outcome(Outcome::Modified, dirstate_node)?
 } else {
 let mtime_looks_clean;
 if let Some(dirstate_mtime) = entry.truncated_mtime() {
-let fs_mtime = TruncatedTimestamp::for_mtime_of(fs_metadata)
+let fs_mtime = TruncatedTimestamp::for_mtime_of(&fs_metadata)
 .expect("OS/libc does not support mtime?");
 // There might be a change in the future if for example the
 // internal clock become off while process run, but this is a
 // case where the issues the user would face
 // would be a lot worse and there is nothing we
 has_ignored_ancestor: bool,
 directory_hg_path: &HgPath,
 fs_entry: &DirEntry,
 ) -> bool {
 let hg_path = directory_hg_path.join(&fs_entry.hg_path);
-let file_type = fs_entry.metadata.file_type();
+let file_or_symlink = fs_entry.is_file() || fs_entry.is_symlink();
-let file_or_symlink = file_type.is_file() || file_type.is_symlink();
+if fs_entry.is_dir() {
-if file_type.is_dir() {
 let is_ignored =
 has_ignored_ancestor || (self.ignore_fn)(&hg_path);
 let traverse_children = if is_ignored {
 // Descendants of an ignored directory are all ignored
 self.options.list_ignored
 // ignored
 self.options.list_unknown || self.options.list_ignored
 };
 if traverse_children {
 let is_at_repo_root = false;
-if let Ok(children_fs_entries) = self.read_dir(
+if let Ok(children_fs_entries) =
-&hg_path,
+self.read_dir(&hg_path, &fs_entry.fs_path, is_at_repo_root)
-&fs_entry.fs_path,
+{
-is_at_repo_root,
-) {
 children_fs_entries.par_iter().for_each(|child_fs_entry| {
 self.traverse_fs_only(
 is_ignored,
 &hg_path,
 child_fs_entry,
 }
 is_ignored
 }
 }
-struct DirEntry {
+/// Since [`std::fs::FileType`] cannot be built directly, we emulate what we
-/// Path as stored in the dirstate
+/// care about.
-hg_path: HgPathBuf,
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum FakeFileType {
+File,
+Directory,
+Symlink,
+}
+impl TryFrom<std::fs::FileType> for FakeFileType {
+type Error = ();
+fn try_from(f: std::fs::FileType) -> Result<Self, Self::Error> {
+if f.is_dir() {
+Ok(Self::Directory)
+} else if f.is_file() {
+Ok(Self::File)
+} else if f.is_symlink() {
+Ok(Self::Symlink)
+} else {
+// Things like FIFO etc.
+Err(())
+}
+}
+}
+struct DirEntry<'a> {
+/// Path as stored in the dirstate, or just the filename for optimization.
+hg_path: HgPathCow<'a>,
 /// Filesystem path
-fs_path: PathBuf,
+fs_path: Cow<'a, Path>,
-metadata: std::fs::Metadata,
+/// Lazily computed
-}
+symlink_metadata: Option<std::fs::Metadata>,
+/// Already computed for ergonomics.
-impl DirEntry {
+file_type: FakeFileType,
-/// Returns **unsorted** entries in the given directory, with name and
+}
-/// metadata.
+impl<'a> DirEntry<'a> {
+/// Returns **unsorted** entries in the given directory, with name,
+/// metadata and file type.
 ///
 /// If a `.hg` sub-directory is encountered:
 ///
 /// * At the repository root, ignore that sub-directory
 /// * Elsewhere, we’re listing the content of a sub-repo. Return an empty
 let at_cwd = path == Path::new("");
 let read_dir_path = if at_cwd { Path::new(".") } else { path };
 let mut results = Vec::new();
 for entry in read_dir_path.read_dir()? {
 let entry = entry?;
-let metadata = match entry.metadata() {
+let file_type = match entry.file_type() {
 Ok(v) => v,
 Err(e) => {
 // race with file deletion?
 if e.kind() == std::io::ErrorKind::NotFound {
 continue;
 // FIXME don't do this when cached
 if file_name == ".hg" {
 if is_at_repo_root {
 // Skip the repo’s own .hg (might be a symlink)
 continue;
-} else if metadata.is_dir() {
+} else if file_type.is_dir() {
 // A .hg sub-directory at another location means a subrepo,
 // skip it entirely.
 return Ok(Vec::new());
 }
 }
 let full_path = if at_cwd {
 file_name.clone().into()
 } else {
 entry.path()
 };
-let base_name = get_bytes_from_os_string(file_name).into();
+let filename =
+Cow::Owned(get_bytes_from_os_string(file_name).into());
+let file_type = match FakeFileType::try_from(file_type) {
+Ok(file_type) => file_type,
+Err(_) => continue,
+};
 results.push(DirEntry {
-hg_path: base_name,
+hg_path: filename,
-fs_path: full_path,
+fs_path: Cow::Owned(full_path.to_path_buf()),
-metadata,
+symlink_metadata: None,
+file_type,
 })
 }
 Ok(results)
+}
+fn symlink_metadata(&self) -> Result<std::fs::Metadata, std::io::Error> {
+match &self.symlink_metadata {
+Some(meta) => Ok(meta.clone()),
+None => std::fs::symlink_metadata(&self.fs_path),
+}
+}
+fn is_dir(&self) -> bool {
+self.file_type == FakeFileType::Directory
+}
+fn is_file(&self) -> bool {
+self.file_type == FakeFileType::File
+}
+fn is_symlink(&self) -> bool {
+self.file_type == FakeFileType::Symlink
 }
 }
 /// Return the `mtime` of a temporary file newly-created in the `.hg` directory
 /// of the give repository.

Mercurial > hg

comparison rust/hg-core/src/dirstate_tree/status.rs @ 49568:da48f170d203