tests/fsmonitor-run-tests.py

update: add a Rust fast-path when updating from null (and clean)

This case is easy to detect and we have all we need to generate a valid
working copy and dirstate entirely in Rust, which speeds things up
considerably:

On my machine updating a repo of ~300k files goes from 10.00s down to 4.2s,
all while consuming 50% less system time, with all caches hot.
Something to note is that further improvements will probably happen
with the upcoming `InnerRevlog` series that does smarter
mmap hanlding, especially for filelogs.

Here are benchmark numbers on a machine with only 4 cores (and no SMT enabled)

```
### data-env-vars.name               = heptapod-public-2024-03-25-ds2-pnm
  # benchmark.name                   = hg.command.update
  # bin-env-vars.hg.py-re2-module    = default
  # bin-env-vars.hg.changeset.node   = <this change>
  # benchmark.variants.atomic-update = no
  # benchmark.variants.scenario      = null-to-tip
  # benchmark.variants.worker        = default
default: 5.328762  ~~~~~
rust: 1.308654  (-75.44%, -4.02)
### data-env-vars.name               = mercurial-devel-2024-03-22-ds2-pnm
  # benchmark.name                   = hg.command.update
  # bin-env-vars.hg.py-re2-module    = default
  # bin-env-vars.hg.changeset.node   = <this change>
  # benchmark.variants.atomic-update = no
  # benchmark.variants.scenario      = null-to-tip
  # benchmark.variants.worker        = default
default: 1.693271  ~~~~~
rust: 1.151053  (-32.02%, -0.54)
### data-env-vars.name               = mozilla-unified-2024-03-22-ds2-pnm
  # benchmark.name                   = hg.command.update
  # bin-env-vars.hg.py-re2-module    = default
  # bin-env-vars.hg.changeset.node   = <this change>
  # benchmark.variants.atomic-update = no
  # benchmark.variants.scenario      = null-to-tip
  # benchmark.variants.worker        = default
default: 38.901613  ~~~~~
rust: 11.637880 (-70.08%, -27.26)
### data-env-vars.name               = netbsd-xsrc-public-2024-09-19-ds2-pnm
  # benchmark.name                   = hg.command.update
  # bin-env-vars.hg.py-re2-module    = default
  # bin-env-vars.hg.changeset.node   = <this change>
  # benchmark.variants.atomic-update = no
  # benchmark.variants.scenario      = null-to-tip
  # benchmark.variants.worker        = default
default: 4.793727  ~~~~~
rust: 1.505905  (-68.59%, -3.29)
```

#!/usr/bin/env python3

# fsmonitor-run-tests.py - Run Mercurial tests with fsmonitor enabled
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# This is a wrapper around run-tests.py that spins up an isolated instance of
# Watchman and runs the Mercurial tests against it. This ensures that the global
# version of Watchman isn't affected by anything this test does.


import argparse
import contextlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import uuid

osenvironb = getattr(os, 'environb', os.environ)

if sys.version_info > (3, 5, 0):
    PYTHON3 = True

    def _sys2bytes(p):
        return p.encode('utf-8')

elif sys.version_info >= (3, 0, 0):
    print(
        '%s is only supported on Python 3.5+ and 2.7, not %s'
        % (sys.argv[0], '.'.join(str(v) for v in sys.version_info[:3]))
    )
    sys.exit(70)  # EX_SOFTWARE from `man 3 sysexit`
else:
    PYTHON3 = False

    # In python 2.x, path operations are generally done using
    # bytestrings by default, so we don't have to do any extra
    # fiddling there. We define the wrapper functions anyway just to
    # help keep code consistent between platforms.
    def _sys2bytes(p):
        return p


def getparser():
    """Obtain the argument parser used by the CLI."""
    parser = argparse.ArgumentParser(
        description='Run tests with fsmonitor enabled.',
        epilog='Unrecognized options are passed to run-tests.py.',
    )
    # - keep these sorted
    # - none of these options should conflict with any in run-tests.py
    parser.add_argument(
        '--keep-fsmonitor-tmpdir',
        action='store_true',
        help='keep temporary directory with fsmonitor state',
    )
    parser.add_argument(
        '--watchman',
        help='location of watchman binary (default: watchman in PATH)',
        default='watchman',
    )

    return parser


@contextlib.contextmanager
def watchman(args):
    basedir = tempfile.mkdtemp(prefix='hg-fsmonitor')
    try:
        # Much of this configuration is borrowed from Watchman's test harness.
        cfgfile = os.path.join(basedir, 'config.json')
        # TODO: allow setting a config
        with open(cfgfile, 'w') as f:
            f.write(json.dumps({}))

        logfile = os.path.join(basedir, 'log')
        clilogfile = os.path.join(basedir, 'cli-log')
        if os.name == 'nt':
            sockfile = '\\\\.\\pipe\\watchman-test-%s' % uuid.uuid4().hex
        else:
            sockfile = os.path.join(basedir, 'sock')
        pidfile = os.path.join(basedir, 'pid')
        statefile = os.path.join(basedir, 'state')

        argv = [
            args.watchman,
            '--sockname',
            sockfile,
            '--logfile',
            logfile,
            '--pidfile',
            pidfile,
            '--statefile',
            statefile,
            '--foreground',
            '--log-level=2',  # debug logging for watchman
        ]

        envb = osenvironb.copy()
        envb[b'WATCHMAN_CONFIG_FILE'] = _sys2bytes(cfgfile)
        with open(clilogfile, 'wb') as f:
            proc = subprocess.Popen(
                argv, env=envb, stdin=None, stdout=f, stderr=f
            )
            try:
                yield sockfile
            finally:
                proc.terminate()
                proc.kill()
    finally:
        if args.keep_fsmonitor_tmpdir:
            print('fsmonitor dir available at %s' % basedir)
        else:
            shutil.rmtree(basedir, ignore_errors=True)


def run():
    parser = getparser()
    args, runtestsargv = parser.parse_known_args()

    with watchman(args) as sockfile:
        osenvironb[b'WATCHMAN_SOCK'] = _sys2bytes(sockfile)
        # Indicate to hghave that we're running with fsmonitor enabled.
        osenvironb[b'HGFSMONITOR_TESTS'] = b'1'

        runtestdir = os.path.dirname(__file__)
        runtests = os.path.join(runtestdir, 'run-tests.py')
        blacklist = os.path.join(runtestdir, 'blacklists', 'fsmonitor')

        runtestsargv.insert(0, runtests)
        runtestsargv.extend(
            [
                '--extra-config',
                'extensions.fsmonitor=',
                # specify fsmonitor.mode=paranoid always in order to force
                # fsmonitor extension execute "paranoid" code path
                #
                # TODO: make fsmonitor-run-tests.py accept specific options
                '--extra-config',
                'fsmonitor.mode=paranoid',
                '--blacklist',
                blacklist,
            ]
        )

        return subprocess.call(runtestsargv)


if __name__ == '__main__':
    sys.exit(run())