changeset 49517:52464a20add0

rhg: parallellize computation of [unsure_is_modified] [unsure_is_modified] is called for every file for which we can't determine its status based on its size and mtime alone. In particular, this happens if the mtime of the file changes without its contents changing. Parallellizing this improves performance significantly when we have many of these files. Here's an example run (on a repo with ~400k files after dropping FS caches) ``` before: real 0m53.901s user 0m27.806s sys 0m31.325s after: real 0m32.017s user 0m34.277s sys 1m26.250s ``` Another example run (a different FS): ``` before: real 3m28.479s user 0m31.800s sys 0m25.324s after: real 0m29.751s user 0m41.814s sys 1m15.387s ```
author Arseniy Alekseyev <aalekseyev@janestreet.com>
date Wed, 05 Oct 2022 15:45:05 -0400
parents 3854eb90629b
children 943509a58d29
files rust/Cargo.lock rust/hg-core/src/revlog/filelog.rs rust/hg-core/src/sparse.rs rust/rhg/Cargo.toml rust/rhg/src/commands/status.rs
diffstat 5 files changed, 38 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/rust/Cargo.lock	Wed Sep 21 10:14:29 2022 -0400
+++ b/rust/Cargo.lock	Wed Oct 05 15:45:05 2022 -0400
@@ -981,6 +981,7 @@
  "lazy_static",
  "log",
  "micro-timer",
+ "rayon",
  "regex",
  "users",
  "which",
--- a/rust/hg-core/src/revlog/filelog.rs	Wed Sep 21 10:14:29 2022 -0400
+++ b/rust/hg-core/src/revlog/filelog.rs	Wed Oct 05 15:45:05 2022 -0400
@@ -17,18 +17,21 @@
 }
 
 impl Filelog {
-    pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> {
+    pub fn open_vfs(
+        store_vfs: &crate::vfs::Vfs<'_>,
+        file_path: &HgPath,
+    ) -> Result<Self, HgError> {
         let index_path = store_path(file_path, b".i");
         let data_path = store_path(file_path, b".d");
-        let revlog = Revlog::open(
-            &repo.store_vfs(),
-            index_path,
-            Some(&data_path),
-            false,
-        )?;
+        let revlog =
+            Revlog::open(store_vfs, index_path, Some(&data_path), false)?;
         Ok(Self { revlog })
     }
 
+    pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> {
+        Self::open_vfs(&repo.store_vfs(), file_path)
+    }
+
     /// The given node ID is that of the file as found in a filelog, not of a
     /// changeset.
     pub fn data_for_node(
--- a/rust/hg-core/src/sparse.rs	Wed Sep 21 10:14:29 2022 -0400
+++ b/rust/hg-core/src/sparse.rs	Wed Oct 05 15:45:05 2022 -0400
@@ -97,7 +97,7 @@
         Includes,
         Excludes,
         None,
-    };
+    }
 
     let mut current = Current::None;
     let mut in_section = false;
--- a/rust/rhg/Cargo.toml	Wed Sep 21 10:14:29 2022 -0400
+++ b/rust/rhg/Cargo.toml	Wed Oct 05 15:45:05 2022 -0400
@@ -22,3 +22,4 @@
 format-bytes = "0.3.0"
 users = "0.11.0"
 which = "4.2.5"
+rayon = "1.5.1"
--- a/rust/rhg/src/commands/status.rs	Wed Sep 21 10:14:29 2022 -0400
+++ b/rust/rhg/src/commands/status.rs	Wed Oct 05 15:45:05 2022 -0400
@@ -29,6 +29,7 @@
 use hg::StatusOptions;
 use hg::{self, narrow, sparse};
 use log::info;
+use rayon::prelude::*;
 use std::io;
 use std::path::PathBuf;
 
@@ -291,16 +292,31 @@
             let manifest = repo.manifest_for_node(p1).map_err(|e| {
                 CommandError::from((e, &*format!("{:x}", p1.short())))
             })?;
-            for to_check in ds_status.unsure {
-                if unsure_is_modified(repo, &manifest, &to_check.path)? {
+            let working_directory_vfs = repo.working_directory_vfs();
+            let store_vfs = repo.store_vfs();
+            let res: Vec<_> = ds_status
+                .unsure
+                .into_par_iter()
+                .map(|to_check| {
+                    unsure_is_modified(
+                        working_directory_vfs,
+                        store_vfs,
+                        &manifest,
+                        &to_check.path,
+                    )
+                    .map(|modified| (to_check, modified))
+                })
+                .collect::<Result<_, _>>()?;
+            for (status_path, is_modified) in res.into_iter() {
+                if is_modified {
                     if display_states.modified {
-                        ds_status.modified.push(to_check);
+                        ds_status.modified.push(status_path);
                     }
                 } else {
                     if display_states.clean {
-                        ds_status.clean.push(to_check.clone());
+                        ds_status.clean.push(status_path.clone());
                     }
-                    fixup.push(to_check.path.into_owned())
+                    fixup.push(status_path.path.into_owned())
                 }
             }
         }
@@ -525,11 +541,12 @@
 /// This meant to be used for those that the dirstate cannot resolve, due
 /// to time resolution limits.
 fn unsure_is_modified(
-    repo: &Repo,
+    working_directory_vfs: hg::vfs::Vfs,
+    store_vfs: hg::vfs::Vfs,
     manifest: &Manifest,
     hg_path: &HgPath,
 ) -> Result<bool, HgError> {
-    let vfs = repo.working_directory_vfs();
+    let vfs = working_directory_vfs;
     let fs_path = hg_path_to_path_buf(hg_path).expect("HgPath conversion");
     let fs_metadata = vfs.symlink_metadata(&fs_path)?;
     let is_symlink = fs_metadata.file_type().is_symlink();
@@ -549,7 +566,7 @@
     if entry.flags != fs_flags {
         return Ok(true);
     }
-    let filelog = repo.filelog(hg_path)?;
+    let filelog = hg::filelog::Filelog::open_vfs(&store_vfs, hg_path)?;
     let fs_len = fs_metadata.len();
     let file_node = entry.node_id()?;
     let filelog_entry = filelog.entry_for_node(file_node).map_err(|_| {