view contrib/heptapod-ci.yml @ 52180:b3e696e879fc stable

ci: have the mac test run if you trigger building the mac wheel

The mac test job now depends on the wheel building. And the wheel building is
manual. So if the mac test job is set to "on_success" if will be "skipped" by
default, and automatically run if the wheel are build.

That is especially handy as we are about to shard that test and that the UI for
manual sharded test sucks.

# Don't run pipelines on branch "merge", since we're fast-forward only.
# Gitlab sees a new branch (since e.g. `topic/stable/my-topic` becomes
# `branch/stable`), but the hash hasn't changed. There is no reason to
# re-run the CI in our case, since we haven't built up any specific automation.
# Right now it's just wasted CI and developer time.
# One can still run the pipeline manually via the web interface,
# like in the case of releases, to make *extra* sure that the actual branch
# has succeeded.
workflow:
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^branch\/.*/ && $CI_PIPELINE_SOURCE != "web"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: never
    - if: $CI_PIPELINE_SOURCE == "push"
      when: always
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH
      when: always

stages:
  - build
  - checks
  - tests
  - platform-compat
  - py-version-compat


image: registry.heptapod.net/mercurial/ci-images/mercurial-core:$HG_CI_IMAGE_TAG

variables:
    PYTHON: python
    HG_CI_IMAGE_TAG: "v2.1"
    # a directory dedicated to creating files and temporary clone
    # with shell runner, its content is not cleaned from one call to the next,
    # so plan for it.
    TMP_WORK_DIR: "${CI_PROJECT_DIR}/../.."

.all:
  # help changing all job at once when debugging
  when: on_success
  # make sure jobs from later steps does not wait for anything implicit before
  # starting.
  needs: []

.build-wheel:
  extends: .all
  image: "registry.heptapod.net/mercurial/ci-images/core-wheel-x86_64-c:v3.0"
  stage: build
  variables:
    WHEEL_TYPE: ""
    FLAVOR: ""
    MERCURIAL_SETUP_FORCE_TRANSLATIONS: "1"
    CI_CLEVER_CLOUD_FLAVOR: "XS"
  before_script:
    - echo $WHEEL_TYPE
    - test -n "$WHEEL_TYPE"
    - echo $FLAVOR
    - mkdir -p wheels/$WHEEL_TYPE/$BUILD_PY_ID
  script:
    - /opt/python/$BUILD_PY_ID/bin/python setup.py bdist_wheel --dist-dir tmp-wheelhouse
    - auditwheel repair tmp-wheelhouse/*.whl -w wheels/$WHEEL_TYPE/$BUILD_PY_ID
  artifacts:
    paths:
      - wheels/$WHEEL_TYPE/$BUILD_PY_ID
    expire_in: 1 week

build-c-wheel:
  extends: .build-wheel
  variables:
    WHEEL_TYPE: "c"
  parallel:
    matrix:
      - BUILD_PY_ID:
          - cp38-cp38
          - cp39-cp39
          - cp310-cp310
          - cp311-cp311
          - cp312-cp312
          - cp313-cp313

.runtests:
    extends: .all
    stage: tests
    variables:
      SHOW_VERSION_OF: "$PYTHON"
      TEST_HGTESTS_ALLOW_NETIO: "0"
      FILTER: ""
      FLAVOR: ""
      RUNTEST_ARGS: ""
    # The runner made a clone as root.
    # We make a new clone owned by user used to run the step.
    before_script:
      - echo "python used, $PYTHON"
      - for tool in $SHOW_VERSION_OF ; do echo '#' version of $tool; $tool --version; done
      - rm -rf "${TMP_WORK_DIR}"/mercurial-ci/  # Clean slate if not using containers
      - hg clone . "${TMP_WORK_DIR}"/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R "${TMP_WORK_DIR}"/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd "${TMP_WORK_DIR}"/mercurial-ci/
      - ls -1 tests/test-check-*.* > "${TMP_WORK_DIR}"/check-tests.txt
    script:
        - echo "$TEST_HGTESTS_ALLOW_NETIO"
        - echo "$RUNTEST_ARGS"
        - echo "$FILTER"
        - echo "$FLAVOR"
        - echo "$WHEEL_TYPE"
        - PORT_START=`expr 19051 + 1009 '*' $CI_CONCURRENT_ID`
        - PORT_ARG="--port $PORT_START"
        - echo $PORT_ARG
        - WHEEL_ARG=""
        - if test -n "$WHEEL_TYPE"; then
             PY_TAG=`$PYTHON -c 'import sys; v=sys.version_info; t=f"cp{v.major}{v.minor}"; print(f"{t}-{t}")'`;
             echo "$PY_TAG";
             test -n "PY_TAG";
             WHEEL="`ls -1 $CI_PROJECT_DIR/wheels/$WHEEL_TYPE/$PY_TAG/*.whl`";
             test -n "$WHEEL";
             echo installing from $WHEEL;
             WHEEL_ARG="--hg-wheel $WHEEL";
             echo disabling flavor as this is currently incompatible with '"--hg-wheel"';
             FLAVOR="";
          else
            echo installing from source;
          fi;
        - HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO"
          "$PYTHON" tests/run-tests.py
            --color=always
            $PORT_ARG
            $WHEEL_ARG
            $FLAVOR
            $FILTER
            $RUNTEST_ARGS;

checks:
    extends: .runtests
    stage: checks
    variables:
        SHOW_VERSION_OF: "$PYTHON black clang-format"
        RUNTEST_ARGS: "--time"
        FILTER: "--test-list ${TMP_WORK_DIR}/check-tests.txt"
        CI_CLEVER_CLOUD_FLAVOR: S

rust-cargo-test:
    extends: .all
    stage: checks
    script:
        - make rust-tests
        - make cargo-clippy
    variables:
        CI_CLEVER_CLOUD_FLAVOR: S

.runtests-no-check:
  extends: .runtests
  variables:
      FILTER: "--blacklist ${TMP_WORK_DIR}/check-tests.txt"
      TEST_HGTESTS_ALLOW_NETIO: "1"

.test-c:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--no-rust"

test-c:
    extends: .test-c
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp311-cp311"
    variables:
        WHEEL_TYPE: "c"

test-pure:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--pure"

test-rust:
    extends: .runtests-no-check
    variables:
        HGWITHRUSTEXT: "cpython"
        FLAVOR: "--rust"

test-rhg:
    extends: .runtests-no-check
    variables:
        HGWITHRUSTEXT: "cpython"
        FLAVOR: "--rust --rhg"

test-chg:
    extends: .runtests-no-check
    variables:
        FLAVOR: "--chg"

# note: we should probably get a full matrix for flavor × py-version, but this
# is a simple start to be able to check if we break the lowest supported
# version (and 3.12 have been giving us various troubles)
test-3.8-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.8
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp38-cp38"

test-3.12-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.12
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp312-cp312"

test-3.12-rust:
    extends: test-rust
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.12

test-3.13-c:
    extends: .test-c
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.13
        WHEEL_TYPE: "c"
    needs:
      - job: build-c-wheel
        parallel:
          matrix:
            - BUILD_PY_ID: "cp313-cp313"

test-3.13-rust:
    extends: test-rust
    stage: py-version-compat
    when: manual  # avoid overloading the CI by default
    variables:
        PYTHON: python3.13

check-pytype:
    extends: test-rust
    stage: checks
    before_script:
      - export PATH="/home/ci-runner/vendor/pyenv/pyenv-2.4.7-adf3c2bccf09cdb81febcfd15b186711a33ac7a8/shims:/home/ci-runner/vendor/pyenv/pyenv-2.4.7-adf3c2bccf09cdb81febcfd15b186711a33ac7a8/bin:$PATH"
      - echo "PATH, $PATH"
      - hg clone . "${TMP_WORK_DIR}"/mercurial-ci/ --noupdate --config phases.publish=no
      - hg -R "${TMP_WORK_DIR}"/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
      - cd "${TMP_WORK_DIR}"/mercurial-ci/
      - make local PYTHON=$PYTHON
      - ./contrib/setup-pytype.sh
    script:
      - echo "Entering script section"
      - sh contrib/check-pytype.sh

# `sh.exe --login` sets a couple of extra environment variables that are defined
# in the MinGW shell, but switches CWD to /home/$username.  The previous value
# is stored in OLDPWD.  Of the added variables, MSYSTEM is crucial to running
# run-tests.py- it is needed to make run-tests.py generate a `python3` script
# that satisfies the various shebang lines and delegates to `py -3`.
.windows-runtests:
    extends: .all
    when: manual  # we don't have any Windows runners anymore at the moment
    stage: platform-compat
    before_script:
      - C:/hgdev/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && ls -1 tests/test-check-*.* > "${TMP_WORK_DIR}"/check-tests.txt'
      # TODO: find/install cvs, bzr, perforce, gpg, sqlite3

    script:
        - echo "Entering script section"
        - echo "python used, $Env:PYTHON"
        - Invoke-Expression "$Env:PYTHON -V"
        - echo "$Env:RUNTEST_ARGS"
        - echo "$Env:TMP"
        - echo "$Env:TEMP"

        - C:/hgdev/MinGW/msys/1.0/bin/sh.exe --login -c 'cd "$OLDPWD" && HGTESTS_ALLOW_NETIO="$TEST_HGTESTS_ALLOW_NETIO" $PYTHON tests/run-tests.py --color=always $RUNTEST_ARGS'
    variables:
      TMP_WORK_DIR: "C:/hgdev/tmp"

windows:
    extends: .windows-runtests
    when: manual
    tags:
      - windows
    variables:
        RUNTEST_ARGS: "-j 8 --blacklist ${TMP_WORK_DIR}/check-tests.txt"
        PYTHON: C:/hgdev/venvs/python39-x64/Scripts/python.exe

windows-pyox:
    extends: .windows-runtests
    when: manual  # pyoxidizer builds seem broken with --no-use-pep517
    tags:
      - windows
    variables:
        RUNTEST_ARGS: "--blacklist ${TMP_WORK_DIR}/check-tests.txt --pyoxidized"
        PYTHON: C:/hgdev/venvs/python39-x64/Scripts/python.exe

macos:
    extends: .test-c
    stage: platform-compat
    tags:
      - macos
    variables:
        WHEEL_TYPE: "c"
    needs:
      - build-c-wheel-macos

# we use CIBW_SKIP="pp*" to prevent the building of pypy wheel that are neither
# needed nor working.
#
# We could use CIBW_BUILD="cp310-*" to only build the Python 3.10 wheel for now as
# this is the only one we need to test. However testing that build work on all version is useful and match what we do with Linux.
#
# The weird directory structure match the one we use for Linux to deal with the
# multiple jobs. (all this might be unnecessary)
build-c-wheel-macos:
    when: manual  # avoid overloading the CI by default
    stage: build
    tags:
      - macos
    script:
      - rm -rf tmp-wheels
      - CIBW_SKIP="pp*" cibuildwheel --output-dir tmp-wheels/
      - for py_version in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313; do
          mkdir -p wheels/c/$py_version/;
          mv tmp-wheels/*$py_version*.whl wheels/c/$py_version/;
        done
      - rm -rf tmp-wheels
    artifacts:
      paths:
        - wheels
      expire_in: 1 week