rust-status: query fs traversal metadata lazily
Currently, any time the status algorithm needs to read a directory from the
filesystem (because the stat-only optimization is not available), it also
stats each directory entry eagerly.
Stat'ing the entries is only needed in a few cases (like when checking
the mtime of a directory for caching): this patch creates a wrapper struct
`DirEntry` that only stats the directory entry it represents when needed.
Excerpt of an `strace` before this change on Mozilla Central:
```
openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
newfstatat(3, "", {st_mode=S_IFDIR|0755, st_size=3540, ...}, AT_EMPTY_PATH) = 0
getdents64(3, 0x
55dc970bd440 /* 139 entries */, 32768) = 5072
statx(3, ".hg", AT_STATX_SYNC_AS_STAT|AT_SYMLINK_NOFOLLOW, STATX_ALL, {stx_mask=STATX_ALL|STATX_MNT_ID, stx_attributes=0, stx_mode=S_IFDIR|0755, stx_size=772, ...}) = 0
[... 135 other successful `statx` calls]
getdents64(3, 0x
55dc970bd440 /* 0 entries */, 32768) = 0
close(3) = 0
```
After this change:
```
openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
newfstatat(3, "", {st_mode=S_IFDIR|0755, st_size=3540, ...}, AT_EMPTY_PATH) = 0
getdents64(3, 0x
561567c10190 /* 139 entries */, 32768) = 5072
getdents64(3, 0x
561567c10190 /* 0 entries */, 32768) = 0
close(3) = 0
```
stages:
- tests
image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG
variables:
PYTHON: python
TEST_HGMODULEPOLICY: "allow"
HG_CI_IMAGE_TAG: "v1.0"
TEST_HGTESTS_ALLOW_NETIO: "0"
.all_template: &all
when: on_success
.runtests_template: &runtests
<<: *all
stage: tests
# The runner made a clone as root.
# We make a new clone owned by user used to run the step.
before_script:
- hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
- hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
- cd /tmp/mercurial-ci/
- ls -1 tests/test-check-*.* > /tmp/check-tests.txt
- black --version
- clang-format --version
script:
- echo "python used, $PYTHON"
- echo "$RUNTEST_ARGS"
- HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS
checks:
<<: *runtests
variables:
RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
PYTHON: python3
rust-cargo-test:
<<: *all
stage: tests
script:
- echo "python used, $PYTHON"
- make rust-tests
variables:
PYTHON: python3
test-c:
<<: *runtests
variables:
RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
PYTHON: python3
TEST_HGMODULEPOLICY: "c"
TEST_HGTESTS_ALLOW_NETIO: "1"
test-pure:
<<: *runtests
variables:
RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
PYTHON: python3
TEST_HGMODULEPOLICY: "py"
test-rust:
<<: *runtests
variables:
HGWITHRUSTEXT: cpython
RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
PYTHON: python3
TEST_HGMODULEPOLICY: "rust+c"
test-rhg:
<<: *runtests
variables:
HGWITHRUSTEXT: cpython
RUNTEST_ARGS: "--rust --rhg --blacklist /tmp/check-tests.txt"
PYTHON: python3
TEST_HGMODULEPOLICY: "rust+c"
test-chg:
<<: *runtests
variables:
PYTHON: python3
RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
TEST_HGMODULEPOLICY: "c"
check-pytype:
extends: .runtests_template
before_script:
- hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
- hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
- cd /tmp/mercurial-ci/
- make local PYTHON=$PYTHON
- $PYTHON -m pip install --user -U libcst==0.3.20 pytype==2022.03.29
script:
- echo "Entering script section"
- sh contrib/check-pytype.sh
variables:
PYTHON: python3
# `sh.exe --login` sets a couple of extra environment variables that are defined
# in the MinGW shell, but switches CWD to /home/$username. The previous value
# is stored in OLDPWD. Of the added variables, MSYSTEM is crucial to running
# run-tests.py- it is needed to make run-tests.py generate a `python3` script
# that satisfies the various shebang lines and delegates to `py -3`.
.window_runtests_template: &windows_runtests
<<: *all
when: manual # we don't have any Windows runners anymore at the moment
stage: tests
before_script:
- C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > C:/Temp/check-tests.txt'
# TODO: find/install cvs, bzr, perforce, gpg, sqlite3
script:
- echo "Entering script section"
- echo "python used, $Env:PYTHON"
- Invoke-Expression "$Env:PYTHON -V"
- Invoke-Expression "$Env:PYTHON -m black --version"
- echo "$Env:RUNTEST_ARGS"
- echo "$Env:TMP"
- echo "$Env:TEMP"
- C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'
windows:
<<: *windows_runtests
tags:
- windows
variables:
TEST_HGMODULEPOLICY: "c"
RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt"
PYTHON: py -3
windows-pyox:
<<: *windows_runtests
tags:
- windows
variables:
TEST_HGMODULEPOLICY: "c"
RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt --pyoxidized"
PYTHON: py -3