From 23bd11f9bed2453c4ce25d4359db91461e34b8fa Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Thu, 9 Apr 2020 15:57:28 +0000 Subject: [PATCH] Allow useful CI to run in forks * Move the fast jobs with no dependencies to the first stage. * Remove the global KTH-specific job runner tag from jobs in the pre-build stage. * Use the `pre-build` stage as the dependency for all later stages, rather than the `simple-build` job, specifically. * Convert rule sets to new *rules* syntax. * Use '$CI_PROJECT_NAMESPACE == "gromacs"' to distinguish jobs created with access to GROMACS GitLab infrastructure. Fixes #3458 --- admin/gitlab-ci/archive.gitlab-ci.yml | 8 +- admin/gitlab-ci/documentation.gitlab-ci.yml | 5 -- admin/gitlab-ci/global.gitlab-ci.yml | 78 ++++++++++++------- admin/gitlab-ci/gromacs.gitlab-ci.yml | 67 +++------------- admin/gitlab-ci/lint.gitlab-ci.yml | 28 ++----- admin/gitlab-ci/python-gmxapi.gitlab-ci.yml | 16 ++++ .../sample_restraint-regression.gitlab-ci.yml | 8 ++ .../gitlab-ci/sample_restraint.gitlab-ci.yml | 6 ++ docs/dev-manual/gitlab.rst | 27 ++++++- 9 files changed, 122 insertions(+), 121 deletions(-) diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index 9e2be222aa..1d009808ee 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -112,9 +112,7 @@ archive:configure:nightly: - .docs:configure - .rules:nightly-not-for-release stage: nightly-configure - needs: - - job: simple-build - artifacts: false + dependencies: [] variables: BUILD_DIR: build-package CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None @@ -192,8 +190,6 @@ archive:build: - .rules:nightly-not-for-release stage: nightly-build needs: - - job: simple-build - artifacts: false - job: archive:configure:nightly artifacts: true variables: @@ -254,8 +250,6 @@ archive:nightly-webpage: - .rules:nightly-not-for-release stage: nightly-deploy needs: - - job: simple-build - artifacts: false - job: linkchecker artifacts: false - job: webpage:build diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 581bddded9..61d1deda41 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -177,9 +177,6 @@ docs:configure: extends: - .docs:configure - .rules:not-for-release - needs: - - job: simple-build - artifacts: false variables: BUILD_DIR: build-docs CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on @@ -209,8 +206,6 @@ docs:build: - .rules:not-for-release stage: documentation needs: - - job: simple-build - artifacts: false - job: docs:configure artifacts: true variables: diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml index 3646ffbfc2..bbb0716c7d 100644 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -27,40 +27,64 @@ - export CCACHE_BASEDIR=${PWD} - export CCACHE_DIR=${PWD}/ccache +# Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE +# is set. Includes non-gromacs projects. Note that jobs using this rule are +# eligible to run on non-gromacs project infrastructure, and should therefore +# override the default *tag* parameter to exclude tags specific to the GROMACS +# GitLab Runner infrastructure. I.e. in the job definition, set `tags: []` +.rules:basic: + rules: + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: always + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + +# Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE +# is set. Excludes non-gromacs projects. +.rules:not-for-release: + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: always + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + # Rule to run a job only in nightly release-preparation pipelines. # Checks if the GROMACS_RELEASE variable was set (typically through the GitLab web interface). -# Excludes merge_requests +# Excludes merge_requests and non-gromacs projects. # TODO: Update to *rules* syntax. .rules:nightly-only-for-release: - only: - refs: - - web - - schedules - variables: - - $GROMACS_RELEASE + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE && $CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$GROMACS_RELEASE && $CI_PIPELINE_SOURCE == "schedule"' + when: always # Jobs that run on schedules, but not for merge requests or when GROMACS_RELEASE -# is set. +# is set. Excludes non-gromacs projects. .rules:nightly-not-for-release: - only: - - web - - schedules - except: - variables: - - $GROMACS_RELEASE - -# Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE -# is set. -.rules:not-for-release: - only: - refs: - - web - - pushes - - schedules - - merge_requests - except: - variables: - - $GROMACS_RELEASE + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule"' + when: always # Behavioral templates diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index c3b61ab1f7..8209e2125e 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -6,8 +6,9 @@ simple-build: - .use-ccache - .variables:default - .use-gcc7 - - .rules:not-for-release + - .rules:basic stage: pre-build + tags: [] variables: KUBERNETES_CPU_LIMIT: 8 KUBERNETES_CPU_REQUEST: 4 @@ -220,9 +221,7 @@ gromacs:gcc-7:configure: - .gromacs:base:configure - .use-gcc7 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] gromacs:gcc-7:release:configure: extends: @@ -242,9 +241,7 @@ gromacs:gcc-7-simd-sse41:configure: - .gromacs:base:configure - .use-gcc7 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] variables: CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" @@ -253,9 +250,7 @@ gromacs:gcc-7-double:configure: - .gromacs:base:configure - .use-gcc7 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] variables: CMAKE_PRECISION_OPTIONS: -DGMX_DOUBLE=ON @@ -278,9 +273,7 @@ gromacs:gcc-8:configure: - .gromacs:base:configure - .use-gcc8 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] gromacs:gcc-8:release:configure: extends: @@ -304,9 +297,7 @@ gromacs:gcc-8-cuda-10.2:configure: variables: CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. - needs: - - job: simple-build - artifacts: false + dependencies: [] # For version of clang before 7, we need to always provide the full version # Use something like this instead in the download script: @@ -316,9 +307,7 @@ gromacs:clang-6:configure: - .gromacs:base:configure - .use-clang6 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] variables: COMPILER_MAJOR_VERSION: "6.0" CMAKE_MPI_OPTIONS: -DGMX_OPENMP=OFF @@ -328,9 +317,7 @@ gromacs:clang-7:configure: - .gromacs:base:configure - .use-clang7 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] gromacs:clang-7:release:configure: extends: @@ -350,9 +337,7 @@ gromacs:clang-8:configure: - .gromacs:base:configure - .use-clang8 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] gromacs:clang-8:release:configure: extends: @@ -397,8 +382,6 @@ gromacs:gcc-7:build: - .use-gcc7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7:configure artifacts: true @@ -424,8 +407,6 @@ gromacs:gcc-7-simd-sse41:build: - .use-gcc7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7-simd-sse41:configure artifacts: true @@ -437,8 +418,6 @@ gromacs:gcc-7-double:build: - .use-gcc7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7-double:configure artifacts: true @@ -464,8 +443,6 @@ gromacs:gcc-8:build: - .use-gcc8 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-8:configure artifacts: true @@ -495,8 +472,6 @@ gromacs:gcc-8-cuda-10.2:build: CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-8-cuda-10.2:configure artifacts: true @@ -508,8 +483,6 @@ gromacs:clang-6:build: - .use-clang6 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:clang-6:configure artifacts: true @@ -521,8 +494,6 @@ gromacs:clang-7:build: - .use-clang7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:clang-7:configure artifacts: true @@ -548,8 +519,6 @@ gromacs:clang-8:build: - .use-clang8 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:clang-8:configure artifacts: true @@ -613,8 +582,6 @@ gromacs:gcc-7:test: - .use-gcc7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7:build - job: regressiontests:prepare @@ -639,8 +606,6 @@ gromacs:gcc-7-simd-sse41:test: - .use-gcc7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7-simd-sse41:build - job: regressiontests:prepare @@ -652,8 +617,6 @@ gromacs:gcc-7-double:test: variables: REGRESSIONTEST_DOUBLE: "-double" needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-7-double:build - job: regressiontests:prepare @@ -678,8 +641,6 @@ gromacs:gcc-8:test: - .use-gcc8 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-8:build - job: regressiontests:prepare @@ -713,8 +674,6 @@ gromacs:gcc-8-cuda-10.2:test: CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. needs: - - job: simple-build - artifacts: false - job: gromacs:gcc-8-cuda-10.2:build - job: regressiontests:prepare @@ -726,8 +685,6 @@ gromacs:clang-6:test: variables: REGRESSIONTEST_OMP_RANK_NUMBER: 0 needs: - - job: simple-build - artifacts: false - job: gromacs:clang-6:build - job: regressiontests:prepare @@ -737,8 +694,6 @@ gromacs:clang-7:test: - .use-clang7 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:clang-7:build - job: regressiontests:prepare @@ -763,8 +718,6 @@ gromacs:clang-8:test: - .use-clang8 - .rules:not-for-release needs: - - job: simple-build - artifacts: false - job: gromacs:clang-8:build - job: regressiontests:prepare diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index d75648599f..828b9a3174 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -5,9 +5,7 @@ clang-tidy:configure: - .gromacs:base:configure - .use-clang7 - .rules:not-for-release - needs: - - job: simple-build - artifacts: false + dependencies: [] variables: BUILD_DIR: build-clang-tidy CMAKE_EXTRA_OPTIONS: -DCLANG_TIDY=clang-tidy-$COMPILER_MAJOR_VERSION -DGMX_CLANG_TIDY=ON -DGMX_COMPILER_WARNINGS=ON @@ -24,8 +22,6 @@ clang-tidy:build: - .rules:not-for-release stage: source-check needs: - - job: simple-build - artifacts: false - job: clang-tidy:configure artifacts: true variables: @@ -34,16 +30,12 @@ clang-tidy:build: clang-format: extends: - - .gromacs:base:build - .variables:default - .use-clang7 - - .rules:not-for-release + - .rules:basic cache: {} - stage: source-check - when: always - needs: - - job: simple-build - artifacts: false + stage: pre-build + tags: [] variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -62,16 +54,12 @@ clang-format: copyright-check: extends: - - .gromacs:base:build - .variables:default - .use-clang7 - - .rules:not-for-release + - .rules:basic cache: {} - stage: source-check - when: always - needs: - - job: simple-build - artifacts: false + stage: pre-build + tags: [] variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -100,8 +88,6 @@ check-source: cache: {} stage: source-check needs: - - job: simple-build - artifacts: false - job: docs:build artifacts: true variables: diff --git a/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml index 917c64fabd..ac72f92383 100644 --- a/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml +++ b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml @@ -34,6 +34,8 @@ gmxapi-0.1:clang-8:py-3.5.9: extends: - .gmxapi-0.1:clang-8 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_1' when: always @@ -50,6 +52,8 @@ gmxapi-0.1:clang-8:py-3.6.10: extends: - .gmxapi-0.1:clang-8 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_1' when: always @@ -66,6 +70,8 @@ gmxapi-0.1:clang-8:py-3.7.7: extends: - .gmxapi-0.1:clang-8 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_1' when: always @@ -82,6 +88,8 @@ gmxapi-0.1:clang-8:py-3.8.2: extends: - .gmxapi-0.1:clang-8 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_1' when: always @@ -100,6 +108,8 @@ gmxapi-0.1:clang-8:py-3.8.2: - .variables:default - .use-clang8 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -134,6 +144,8 @@ gmxapi-0.2:clang-8:gmx2021:py-3.6.10: extends: - .gmxapi-0.2:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -150,6 +162,8 @@ gmxapi-0.2:clang-8:gmx2021:py-3.7.7: extends: - .gmxapi-0.2:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -166,6 +180,8 @@ gmxapi-0.2:clang-8:gmx2021:py-3.8.2: extends: - .gmxapi-0.2:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always diff --git a/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml index 4f1dcdbef7..29e59af784 100644 --- a/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml +++ b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml @@ -31,6 +31,8 @@ sample_restraint:clang-8:gmx2020:3.5.9: - .sample_restraint:clang-8:gmx2020 # Run only for changes affecting the python_packaging subdirectory of the release-2020 branch. rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -52,6 +54,8 @@ sample_restraint:clang-8:gmx2020:py-3.6.10: - .sample_restraint:clang-8:gmx2020 # Run only for changes affecting the python_packaging subdirectory of the release-2020 branch. rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -73,6 +77,8 @@ sample_restraint:clang-8:gmx2020:py-3.7.7: - .sample_restraint:clang-8:gmx2020 # Run only for changes affecting the python_packaging subdirectory of the release-2020 branch. rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -94,6 +100,8 @@ sample_restraint:clang-8:gmx2020:py-3.8.2: - .sample_restraint:clang-8:gmx2020 # Run only for changes affecting the python_packaging subdirectory of the release-2020 branch. rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always diff --git a/admin/gitlab-ci/sample_restraint.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml index 9a1985eb7a..e0b02b52ca 100644 --- a/admin/gitlab-ci/sample_restraint.gitlab-ci.yml +++ b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml @@ -26,6 +26,8 @@ sample_restraint:clang-8:gmx2021:py-3.6.10: extends: - .sample_restraint:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -42,6 +44,8 @@ sample_restraint:clang-8:gmx2021:py-3.7.7: extends: - .sample_restraint:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always @@ -58,6 +62,8 @@ sample_restraint:clang-8:gmx2021:py-3.8.2: extends: - .sample_restraint:clang-8:gmx2021 rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never # Allow explicit trigger through web interface. - if: '$GMX_GMXAPI_0_2' when: always diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst index 19894addce..8964a4b81c 100644 --- a/docs/dev-manual/gitlab.rst +++ b/docs/dev-manual/gitlab.rst @@ -1,9 +1,6 @@ GitLab ====== -|Gromacs| is transitioning to GitLab for source code management, issue tracking, -and integrated automation for testing and documentation. - The repository contains DockerFiles and GitLab Runner configuration files to support automated testing and documentation builds. General information on configuring GitLab CI pipelines can be found @@ -15,7 +12,7 @@ Configuration templates are found in the files in the :file:`admin/ci-templates/` directory. Docker images used by GitLab Runner are available on `Docker Hub `__. -Images are (re)built manually from DockerFiles in :file:`admin/dockerfiles`. +Images are (re)built manually using details in :file:`admin/containers`. This documentation is incomplete, pending resolution of :issue:`3275`. @@ -28,6 +25,15 @@ Pipeline execution .. todo:: Comment on the number of pipelines that can be or which are likely to be running at the same time. +.. note:: + + Full automated testing is only available for merge requests originating from + branches of the main https://gitlab.com/gromacs/gromacs repository. + GitLab CI pipelines created for forked repositories will include fewer jobs + in the testing pipeline. Non-trivial merge requests may need to be issued + from a branch in the ``gromacs`` project namespace in order to receive + sufficient testing before acceptance. + Configuration files ~~~~~~~~~~~~~~~~~~~ @@ -84,6 +90,13 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions. To reduce errors and unexpected behavior, restrict usage of these controls to regular job definitions (don't use in "hidden" or parent jobs). + tags + By `default `__, + jobs require the ``k8s-scilifelab`` tag, which identifies Runners in the + |Gromacs| infrastructure. A small number of jobs in the first pipeline + stage override the default with an empty tag list so that all GitLab + users can run basic tests in their forked project. + variables Many job definitions will add or override keys in *variables*. Refer to `GitLab `__ @@ -159,6 +172,12 @@ or passed along to the environment of executed commands. Other important variable keys are as follows. .. glossary:: + CI_PROJECT_NAMESPACE + Distinguishes pipelines created for repositories in the ``gromacs`` + GitLab project space. May be used to pre-screen jobs to determine + whether |Gromacs| GitLab infrastructure is available to the pipeline + before the job is created. + COMPILER_MAJOR_VERSION Integer version number provided by toolchain mix-in for convenience and internal use. -- 2.22.0