contrib/heptapod-ci.yml

clone: properly exclude rev-branch-cache from post clone cache warming

When adding "CACHE_REV_BRANCH" to "CACHES_ALL" in e51161b12c7e, I did not
expected it to impact the clone steps. However the "CACHES_POST_CLONE" set is
created rather creatively. (we should fix that, but not on stable)

The benchmark caught a quite significant slowdown one hardlink and ssh-stream
clones. Such slow down can be reduced to around ~5% by fully warming the cache
before the clone. However keeping this expensive step away from the clone
operation fully fix the slowdown and preserve the initial intend.


Example slowdow for hardlink clone

### benchmark.name                               = hg.command.clone
  # bin-env-vars.hg.flavor                       = default
  # bin-env-vars.hg.py-re2-module                = default
  # benchmark.variants.explicit-rev              = none
  # benchmark.variants.issue6528                 = default
  # benchmark.variants.protocol                  = local-hardlink
  # benchmark.variants.pulled-delta-reuse-policy = default
  # benchmark.variants.resource-usage            = default
  # benchmark.variants.validate                  = default
 ## data-env-vars.name = netbeans-2018-08-01-zstd-sparse-revlog
6.8.2:  19.799752
6.9rc0: 29.017493  (+46.55%, +9.22)
after:  19.929341

 ## data-env-vars.name = mercurial-public-2018-08-01-zstd-sparse-revlog
6.8.2:  0.468020
6.9rc0: 1.701294 (+263.51%, +1.23)
after:  0.471934

 ## data-env-vars.name = pypy-2024-03-22-zstd-sparse-revlog
6.8.2:  2.397564
6.9rc0: 5.666641 (+137.41%, +3.28)
after:  2.428085

# Don't run pipelines on branch "merge", since we're fast-forward only.
# Gitlab sees a new branch (since e.g. `topic/stable/my-topic` becomes
# `branch/stable`), but the hash hasn't changed. There is no reason to
# re-run the CI in our case, since we haven't built up any specific automation.
# Right now it's just wasted CI and developer time.
# One can still run the pipeline manually via the web interface,
# like in the case of releases, to make *extra* sure that the actual branch
# has succeeded.
workflow:
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^branch\/.*/ && $CI_PIPELINE_SOURCE != "web"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: never
    - if: $CI_PIPELINE_SOURCE == "push"
      when: always
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH
      when: always

stages:
  - build
  - checks
  - tests
  - platform-compat
  - py-version-compat
  - upload


image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG

variables:
    PYTHON: python
    HG_CI_IMAGE_TAG: "v2.1"
    # a directory dedicated to creating files and temporary clone
    # with shell runner, its content is not cleaned from one call to the next,
    # so plan for it.
    TMP_WORK_DIR: "${CI_PROJECT_DIR}/../.."
    # we use CIBW_SKIP="pp*" to prevent the building of pypy wheel that are neither
    # needed nor working.
    CIBW_SKIP: "pp*"

.all:
  # help changing all job at once when debugging
  when: on_success
  # make sure jobs from later steps does not wait for anything implicit before
  # starting.
  needs: []

.build-wheel:
  extends: .all
  image: "registry.heptapod.net/mercurial/ci-images/core-wheel-x86_64-c:v3.0"
  stage: build
  variables:
    WHEEL_TYPE: ""
    FLAVOR: ""
    MERCURIAL_SETUP_FORCE_TRANSLATIONS: "1"
    CI_CLEVER_CLOUD_FLAVOR: "XS"
  script:
    - PLATFORM=`/opt/python/cp313-cp313/bin/python -c 'import sys; print(sys.platform)'`
    - echo $WHEEL_TYPE
    - test -n "$WHEEL_TYPE"
    - echo $FLAVOR
    - mkdir -p wheels/$PLATFORM/$WHEEL_TYPE/$BUILD_PY_ID
    - contrib/build-one-linux-wheel.sh $BUILD_PY_ID wheels/$PLATFORM/$WHEEL_TYPE/$BUILD_PY_ID
  artifacts:
    paths:
      - wheels/
    expire_in: 1 week

build-c-wheel:
  extends: .build-wheel
  variables:
    WHEEL_TYPE: "c"
  parallel:
    matrix:
      - BUILD_PY_ID:
          - cp38-cp38
          - cp39-cp39
          - cp310-cp310
          - cp311-cp311
          - cp312-cp312
          - cp313-cp313

.runtests:
    extends: .all
    stage: tests
    variables:
      SHOW_VERSION_OF: "$PYTHON"
      TEST_HGTESTS_ALLOW_NETIO: "0"
      FILTER: ""
      FLAVOR: ""
      RUNTEST_ARGS: ""
    # The runner made a clone as root.
    # We make a new clone owned by user used to run the step.
    before_script:
      - echo "python used, $PYTHON"
      - for tool in $SHOW_VERSION_OF ; do echo '#' version of $tool; $tool --version; done
      - rm -rf "${TMP_WORK_DIR}"/mercurial-ci/  # Clean slate if not using containers
      - hg clone . "${TMP_WORK_DIR}"/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R "${TMP_WORK_DIR}"/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd "${TMP_WORK_DIR}"/mercurial-ci/
      - ls -1 tests/test-check-*.* > "${TMP_WORK_DIR}"/check-tests.txt
    script:
        - echo "$TEST_HGTESTS_ALLOW_NETIO"
        - echo "$RUNTEST_ARGS"
        - echo "$FILTER"
        - echo "$FLAVOR"
        - echo "$WHEEL_TYPE"
        - PORT_START=`expr 19051 + 1009 '*' $CI_CONCURRENT_ID`
        - PORT_ARG="--port $PORT_START"
        - echo $PORT_ARG
        - PLATFORM=`$PYTHON -c 'import sys; print(sys.platform)'`
        - echo $PLATFORM
        - WHEEL_ARG=""
        - SHARDING_ARGS=""
        - if test -n "$WHEEL_TYPE"; then
             PY_TAG=`$PYTHON -c 'import sys; v=sys.version_info; t=f"cp{v.major}{v.minor}"; print(f"{t}-{t}")'`;
             echo "$PY_TAG";
             test -n "PY_TAG";
             WHEEL="`ls -1 $CI_PROJECT_DIR/wheels/$PLATFORM/$WHEEL_TYPE/$PY_TAG/*.whl`";
             test -n "$WHEEL";
             echo installing from $WHEEL;
             WHEEL_ARG="--hg-wheel $WHEEL";
             echo disabling flavor as this is currently incompatible with '"--hg-wheel"';
             FLAVOR="";
          else
            echo installing from source;
          fi;
        - if [ -n "$CI_NODE_INDEX" ]; then 
            echo "Running the test in multiple shard - [$CI_NODE_INDEX/$CI_NODE_TOTAL]";
            SHARDING_ARGS="--shard-index $CI_NODE_INDEX --shard-total $CI_NODE_TOTAL";
            echo "sharding... $SHARDING_ARGS";
          fi
        - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO"
          "$PYTHON" tests/run-tests.py
            --color=always
            $PORT_ARG
            $WHEEL_ARG
            $FLAVOR
            $SHARDING_ARGS
            $FILTER
            $RUNTEST_ARGS;

checks:
    extends: .runtests
    stage: checks
    variables:
        SHOW_VERSION_OF: "$PYTHON black clang-format"
        RUNTEST_ARGS: "--time"
        FILTER: "--test-list ${TMP_WORK_DIR}/check-tests.txt"
        CI_CLEVER_CLOUD_FLAVOR: S

rust-cargo-test:
    extends: .all
    stage: checks
    script:
        - make rust-tests
        - make cargo-clippy
    variables:
        CI_CLEVER_CLOUD_FLAVOR: S

.runtests-no-check:
  extends: .runtests
  variables:
      FILTER: "--blacklist ${TMP_WORK_DIR}/check-tests.txt"
      TEST_HGTESTS_ALLOW_NETIO: "1"

.test-c:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--no-rust"

test-c:
    extends: .test-c
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp311-cp311"
    variables:
        WHEEL_TYPE: "c"

test-pure:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--pure"

test-rust:
    extends: .runtests-no-check
    variables:
        HGWITHRUSTEXT: "cpython"
        FLAVOR: "--rust"

test-rhg:
    extends: .runtests-no-check
    variables:
        HGWITHRUSTEXT: "cpython"
        FLAVOR: "--rust --rhg"

test-chg:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--chg"

# note: we should probably get a full matrix for flavor × py-version, but this
# is a simple start to be able to check if we break the lowest supported
# version (and 3.12 have been giving us various troubles)
test-3.8-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.8
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp38-cp38"

test-3.12-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.12
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp312-cp312"

test-3.12-rust:
    extends: test-rust
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.12

test-3.13-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.13
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp313-cp313"

test-3.13-rust:
    extends: test-rust
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.13

check-pytype:
    extends: test-rust
    stage: checks
    before_script:
      - export PATH="/home/ci-runner/vendor/pyenv/pyenv-2.4.7-adf3c2bccf09cdb81febcfd15b186711a33ac7a8/shims:/home/ci-runner/vendor/pyenv/pyenv-2.4.7-adf3c2bccf09cdb81febcfd15b186711a33ac7a8/bin:$PATH"
      - echo "PATH, $PATH"
      - hg clone . "${TMP_WORK_DIR}"/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R "${TMP_WORK_DIR}"/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd "${TMP_WORK_DIR}"/mercurial-ci/
      - make local PYTHON=$PYTHON
      - ./contrib/setup-pytype.sh
    script:
      - echo "Entering script section"
      - sh contrib/check-pytype.sh

# `sh.exe --login` sets a couple of extra environment variables that are defined
# in the MinGW shell, but switches CWD to /home/$username.  The previous value
# is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
# run-tests.py- it is needed to make run-tests.py generate a `python3` script
# that satisfies the various shebang lines and delegates to `py -3`.

.windows:
    extends: .all
    when: manual  # we don't have any Windows runners anymore at the moment
    tags:
      - windows
    before_script:
      - C:/hgdev/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > "${TMP_WORK_DIR}"/check-tests.txt'
      # TODO: find/install cvs, bzr, perforce, gpg, sqlite3
    variables:
        PYTHON: C:/hgdev/venvs/python39-x64/Scripts/python.exe


# a dummy job that only serve to trigger the wider windows build
trigger-wheel-windows:
  extends: .all
  # smallest I know of
  image: busybox
  when: manual
  stage: build
  variables:
    GIT_STRATEGY: none
    CI_CLEVER_CLOUD_FLAVOR: "XS"
  script:
    - echo 'let us build some wheels.'

build-c-wheel-windows:
    extends: .windows
    stage: build
    # wait for someone to click on "trigger-wheel-windows"
    when: on_success
    needs:
      - "trigger-wheel-windows"
    variables:
      MERCURIAL_SETUP_FORCE_TRANSLATIONS: "1"
    script:
        - echo "Entering script section"
        - echo "python used, $Env:PYTHON"
        - Invoke-Expression "$Env:PYTHON -V"
        - echo "$Env:RUNTEST_ARGS"
        - echo "$Env:TMP"
        - echo "$Env:TEMP"
        - "C:/hgdev/venvs/python39-x64/Scripts/python.exe -m cibuildwheel --output-dir wheels/win32"
    artifacts:
      paths:
        - wheels
      expire_in: 1 week
    parallel:
      matrix:
        # "cp39" is first as it unlock the tests
        - CIBW_BUILD:
          - "cp39-*"
          - "cp38-*"
          - "cp310-*"
          - "cp311-*"
          - "cp312-*"
          - "cp313-*"
          CIBW_ARCHS:
          - "AMD64"
          - "x86"


.windows-runtests:
    extends: .windows
    stage: platform-compat
    # the UX for manual parallel jobs is quite awful, and the job que depends
    # upon are manual anyway, so we can make this start automatically once the
    # associated wheel is ready.
    when: on_success
    parallel: 20
    script:
        - echo "Entering script section"
        - echo "python used, $Env:PYTHON"
        - Invoke-Expression "$Env:PYTHON -V"
        - echo "$Env:HGTESTS_ALLOW_NETIO"
        - echo "$Env:WHEEL_ARG"
        - echo "$Env:FLAVOR"
        - echo "$Env:FILTER"
        - echo "$Env:RUNTEST_ARGS"
        - echo "$Env:TMP"
        - echo "$Env:TEMP"
        # This test is hanging the worker and not that important, so lets skip
        # it for now
        - C:/hgdev/MinGW/msys/1.0/bin/sh.exe -c 'cd "$OLDPWD" && echo tests/test-clonebundles-autogen.t > $TMP_WORK_DIR/windows-skip.txt'

        - C:/hgdev/MinGW/msys/1.0/bin/sh.exe
          --login -c 'cd "$OLDPWD"
            && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO"
               $PYTHON tests/run-tests.py
               --color=always
               $WHEEL_ARG
               $FLAVOR
               --port `expr 19051 + 1009 "*" $CI_CONCURRENT_ID`
               --shard-index $CI_NODE_INDEX --shard-total $CI_NODE_TOTAL
               $FILTER
               $RUNTEST_ARGS;
          '
    variables:
      WHEEL_ARG: ""
      RUNTEST_ARGS: ""
      FLAVOR: ""
      FILTER: "--blacklist ${TMP_WORK_DIR}/check-tests.txt --blacklist ${TMP_WORK_DIR}/windows-skip.txt"

windows:
    extends: .windows-runtests
    variables:
        RUNTEST_ARGS: "-j 8"
        WHEEL_ARG: "--hg-wheel wheels/win32/mercurial-*-cp39-cp39-win_amd64.whl"
    needs:
      - job: build-c-wheel-windows
        parallel:
          matrix:
            - CIBW_BUILD: "cp39-*"
              CIBW_ARCHS: "AMD64"

windows-pyox:
    extends: .windows-runtests
    when: manual  # pyoxidizer builds seem broken with --no-use-pep517
    variables:
        FLAVOR: "--pyoxidized"

macos:
    extends: .test-c
    stage: platform-compat
    # run the test in multiple shard to help spread the load between concurrent
    # MR as the macos runner is a shell runner there is not startup overhead
    # for tests.
    parallel: 10
    tags:
      - macos
    variables:
        WHEEL_TYPE: "c"
    needs:
      - build-c-wheel-macos

# We could use CIBW_BUILD="cp310-*" to only build the Python 3.10 wheel for now as
# this is the only one we need to test. However testing that build work on all
# version is useful and match what we do with Linux.
#
# CIBW_SKIP is set globally at the start of the file. See comment there.
#
# The weird directory structure match the one we use for Linux to deal with the
# multiple jobs. (all this might be unnecessary)
build-c-wheel-macos:
    when: manual  # avoid overloading the CI by default
    stage: build
    tags:
      - macos
    variables:
      MERCURIAL_SETUP_FORCE_TRANSLATIONS: "1"
    script:
      - PLATFORM=`$PYTHON -c 'import sys; print(sys.platform)'`
      - rm -rf tmp-wheels
      - cibuildwheel --output-dir tmp-wheels/
      - for py_version in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313; do
          mkdir -p wheels/$PLATFORM/c/$py_version/;
          mv tmp-wheels/*$py_version*.whl wheels/$PLATFORM/c/$py_version/;
        done
      - rm -rf tmp-wheels
    artifacts:
      paths:
        - wheels
      expire_in: 1 week

# Upload nightly build wheel on the heptapod registry on test success
#
# At the time this task is added, since the mac wheels are built on shell
# runner, those nightly are not be considered fully secured.
#
# In addition, since any job can upload package, pretty much anyone with CI
# access can upload anything pretending to be any version. To fix it we would
# have to prevent the CI token to upload to the registry and have dedicated
# credential accessible only from protected branches.
upload-wheel-nightly:
  extends: .all
  image: "registry.heptapod.net/mercurial/ci-images/twine:v3.0"
  stage: upload
  # because we don't want to upload only half of a wheel
  interruptible: false
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
      # note that at the time of writing this, this job depends on multiple
      # manual one. So it will not run by default, but will automatically run
      # if the manual jobs are triggered.
      #
      # Also beware that "on_success" will ignore failure of manual test we
      # directly depends on. This currently relevant for the "test-3.x-c"
      # tests.
      when: on_success
    - if: $CI_COMMIT_BRANCH =~ /^topic\/.*/
      when: never
      # if you need to test this, make it
      # when: manual
      # allow_failure: true
  needs:
    - build-c-wheel
    - build-c-wheel-macos
    - build-c-wheel-windows
    - test-c
    - macos
    # if we also requires windows to be happy, reach the "50 needed jobs" limit.
    # So we need some intermediate job to reduce the number.
    # - windows
    - test-3.8-c
    - test-3.12-c
    - test-3.13-c
  # It would be nice to be able to restrict that a bit to protected branch only
  variables:
    TWINE_USERNAME: gitlab-ci-token
    TWINE_PASSWORD: $CI_JOB_TOKEN
  script:
    - twine
      upload
      --verbose
      --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi
      wheels/*/*/*/*.whl
      wheels/*/*.whl