view contrib/heptapod-ci.yml @ 48178:f12a19d03d2c

fix: reduce number of tool executions

By grouping together (path, ctx) pairs according to the inputs they would
provide to fixer tools, we can deduplicate executions of fixer tools to
significantly reduce the amount of time spent running slow tools.

This change does not handle clean files in the working copy, which could still
be deduplicated against the files in the checked out commit. It's a little
harder to do that because the filerev is not available in the workingfilectx
(and it doesn't exist for added files).

Anecdotally, this change makes some real uses cases at Google 10x faster. I
think we were originally hesitant to do this because the benefits weren't
obvious, and implementing it efficiently is kind of tricky. If we simply
memoized the formatter execution function, we would be keeping tons of file
content in memory.

Also included is a regression test for a corner case that I broke with my first
attempt at optimizing this code.

Differential Revision: https://phab.mercurial-scm.org/D11280

stages:
  - tests
  - phabricator

image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG

variables:
    PYTHON: python
    TEST_HGMODULEPOLICY: "allow"
    HG_CI_IMAGE_TAG: "latest"
    TEST_HGTESTS_ALLOW_NETIO: "0"

.all_template: &all
  when: always

.runtests_template: &runtests
    <<: *all
    stage: tests
    # The runner made a clone as root.
    # We make a new clone owned by user used to run the step.
    before_script:
      - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd /tmp/mercurial-ci/
      - ls -1 tests/test-check-*.* > /tmp/check-tests.txt
      - black --version
      - clang-format --version
    script:
        - echo "python used, $PYTHON"
        - echo "$RUNTEST_ARGS"
        - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS

checks-py2:
    <<: *runtests
    variables:
        RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"

checks-py3:
    <<: *runtests
    variables:
        RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
        PYTHON: python3

rust-cargo-test-py2: &rust_cargo_test
    <<: *all
    stage: tests
    script:
        - echo "python used, $PYTHON"
        - make rust-tests

rust-cargo-test-py3:
    stage: tests
    <<: *rust_cargo_test
    variables:
        PYTHON: python3

phabricator-refresh:
    stage: phabricator
    variables:
      DEFAULT_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)"
      STABLE_COMMENT: ":white_check_mark: refresh by Heptapod after a successful CI run (:octopus: :green_heart:)\n⚠ This patch is intended for stable ⚠\n{image https://media.giphy.com/media/nYI8SmmChYXK0/source.gif}"
    script:
      - |
        if [ `hg branch` == "stable" ]; then
            ./contrib/phab-refresh-stack.sh --comment "$STABLE_COMMENT";
        else
            ./contrib/phab-refresh-stack.sh --comment "$DEFAULT_COMMENT";
        fi

test-py2:
    <<: *runtests
    variables:
        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
        TEST_HGMODULEPOLICY: "c"
        TEST_HGTESTS_ALLOW_NETIO: "1"

test-py3:
    <<: *runtests
    variables:
        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
        PYTHON: python3
        TEST_HGMODULEPOLICY: "c"
        TEST_HGTESTS_ALLOW_NETIO: "1"

test-py2-pure:
    <<: *runtests
    variables:
        RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
        TEST_HGMODULEPOLICY: "py"

test-py3-pure:
    <<: *runtests
    variables:
        RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
        PYTHON: python3
        TEST_HGMODULEPOLICY: "py"

test-py2-rust:
    <<: *runtests
    variables:
        HGWITHRUSTEXT: cpython
        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
        TEST_HGMODULEPOLICY: "rust+c"

test-py3-rust:
    <<: *runtests
    variables:
        HGWITHRUSTEXT: cpython
        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
        PYTHON: python3
        TEST_HGMODULEPOLICY: "rust+c"

test-py3-rhg:
    <<: *runtests
    variables:
        HGWITHRUSTEXT: cpython
        RUNTEST_ARGS: "--rust --rhg --blacklist /tmp/check-tests.txt"
        PYTHON: python3
        TEST_HGMODULEPOLICY: "rust+c"

test-py2-chg:
    <<: *runtests
    variables:
        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
        TEST_HGMODULEPOLICY: "c"

test-py3-chg:
    <<: *runtests
    variables:
        PYTHON: python3
        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
        TEST_HGMODULEPOLICY: "c"

check-pytype-py3:
    extends: .runtests_template
    before_script:
      - hg clone . /tmp/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd /tmp/mercurial-ci/
      - make local PYTHON=$PYTHON
      - $PYTHON -m pip install --user -U pytype==2021.04.15
    variables:
        RUNTEST_ARGS: " --allow-slow-tests tests/test-check-pytype.t"
        HGTEST_TIMEOUT: "3600"
        PYTHON: python3
        TEST_HGMODULEPOLICY: "c"

# `sh.exe --login` sets a couple of extra environment variables that are defined
# in the MinGW shell, but switches CWD to /home/$username.  The previous value
# is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
# run-tests.py- it is needed to make run-tests.py generate a `python3` script
# that satisfies the various shebang lines and delegates to `py -3`.
.window_runtests_template: &windows_runtests
    <<: *all
    when: manual  # we don't have any Windows runners anymore at the moment
    stage: tests
    before_script:
      - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > C:/Temp/check-tests.txt'
      # TODO: find/install cvs, bzr, perforce, gpg, sqlite3

    script:
        - echo "Entering script section"
        - echo "python used, $Env:PYTHON"
        - Invoke-Expression "$Env:PYTHON -V"
        - Invoke-Expression "$Env:PYTHON -m black --version"
        - echo "$Env:RUNTEST_ARGS"
        - echo "$Env:TMP"
        - echo "$Env:TEMP"

        - C:/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" HGMODULEPOLICY="$TEST_HGMODULEPOLICY" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'

windows-py3:
    <<: *windows_runtests
    tags:
      - windows
    variables:
        TEST_HGMODULEPOLICY: "c"
        RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt"
        PYTHON: py -3

windows-py3-pyox:
    <<: *windows_runtests
    tags:
      - windows
    variables:
        TEST_HGMODULEPOLICY: "c"
        RUNTEST_ARGS: "--blacklist C:/Temp/check-tests.txt --pyoxidized"
        PYTHON: py -3