From 9ccc92dd3ff548f8d6a1880a168d82529b37d4d7 Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Sat, 21 Mar 2020 21:29:53 +0300 Subject: [PATCH] Extract logic for when CI jobs execute. Introduce `.rules:` mix-ins to assert when jobs should run. Change-Id: I1bd74f347fb8e19ddc49dee2d173520dcb436820 --- admin/gitlab-ci/archive.gitlab-ci.yml | 94 +------ admin/gitlab-ci/documentation.gitlab-ci.yml | 79 ++---- admin/gitlab-ci/global.gitlab-ci.yml | 35 +++ admin/gitlab-ci/gromacs.gitlab-ci.yml | 264 ++++---------------- admin/gitlab-ci/lint.gitlab-ci.yml | 29 +-- docs/dev-manual/gitlab.rst | 39 +++ 6 files changed, 153 insertions(+), 387 deletions(-) diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index 895e2e1347..b2096f00fb 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -3,6 +3,7 @@ prepare-release-version: extends: - .variables:default + - .rules:nightly-only-for-release cache: {} # Docker image uploaded to dockerhub by user eriklindahl # TODO: Get DockerFile for admin/dockerfiles @@ -13,14 +14,6 @@ prepare-release-version: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi - - only: - refs: - - merge_requests - - schedules - - web - variables: - - $GROMACS_RELEASE script: - cmake -P cmake/gmxVersionInfo.cmake &> version.json @@ -34,6 +27,7 @@ prepare-release-version: regressiontests:prepare: extends: - .variables:default + - .rules:not-for-release cache: {} image: biophysics/gcc-gromacs stage: configure-build @@ -42,15 +36,6 @@ regressiontests:prepare: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi - only: - refs: - - web - - pushes - - merge_requests - - schedules - except: - variables: - - $GROMACS_RELEASE # Always clone the default version for this branch, master in this case script: - export REGTESTBRANCH=master @@ -76,6 +61,7 @@ regressiontests:prepare: regressiontests:package: extends: - .variables:default + - .rules:nightly-only-for-release cache: {} # Docker image uploaded to dockerhub by user eriklindahl # TODO: Get DockerFile for admin/dockerfiles @@ -89,13 +75,6 @@ regressiontests:package: needs: - job: prepare-release-version artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE script: - VERSION=`cat version.json | python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` @@ -127,19 +106,13 @@ regressiontests:package: - regressiontests-*.tar.gz archive:configure:nightly: + extends: + - .docs:configure + - .rules:nightly-not-for-release stage: nightly-configure - only: - - web - - triggers - - schedules needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE - extends: - - .docs:configure variables: BUILD_DIR: build-package CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None @@ -147,15 +120,9 @@ archive:configure:nightly: CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF archive:configure:release: - only: - refs: - - web - - triggers - - schedules - variables: - - $GROMACS_RELEASE extends: - .docs:configure + - .rules:nightly-only-for-release variables: BUILD_DIR: build-package CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None @@ -176,12 +143,6 @@ archive:configure:release: - .docs:build # Dockerfiles are from dockerhub, user eriklindahl image: gromacs/ci-docs-clang:master - only: - refs: - - web - - pushes - - schedules - - merge_requests variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 @@ -226,36 +187,24 @@ archive:configure:release: archive:build: extends: - .archive:build + - .rules:nightly-not-for-release stage: nightly-build needs: - job: simple-build artifacts: false - job: archive:configure:nightly artifacts: true - only: - - web - - triggers - - schedules - except: - variables: - - $GROMACS_RELEASE variables: BUILD_DIR: build-package archive:package: extends: - .archive:build + - .rules:nightly-only-for-release stage: release-package needs: - job: archive:configure:release artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE variables: BUILD_DIR: build-package @@ -264,20 +213,13 @@ release-verify: stage: release-verify extends: - .variables:default + - .rules:nightly-only-for-release cache: {} variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi - - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE script: - VERSION=`cat version.json | python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` @@ -307,14 +249,8 @@ release-verify: archive:nightly-webpage: extends: - .webpage:build + - .rules:nightly-not-for-release stage: nightly-deploy - only: - - web - - triggers - - schedules - except: - variables: - - $GROMACS_RELEASE needs: - job: simple-build artifacts: false @@ -334,14 +270,8 @@ archive:nightly-webpage: archive:full-release: extends: - .webpage:build + - .rules:nightly-only-for-release stage: release-deploy - only: - refs: - - web - - triggers - - schedules - variables: - - $GROMACS_RELEASE dependencies: - webpage:build:release - archive:package diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index d620ea5673..581bddded9 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -1,20 +1,14 @@ # Build documentation and web page. .docs:configure: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:master - stage: configure-build extends: - .gromacs:base:configure - .variables:default - .before_script:default - only: - refs: - - web - - pushes - - schedules - - merge_requests + # Built by admin/dockerfiles/ci-docs-clang + # TODO this should be organized more like the current documentation.py script + image: gromacs/ci-docs-clang:master + stage: configure-build variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -72,14 +66,15 @@ - $BUILD_DIR .docs:configure:release: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:master - stage: release-configure extends: - .gromacs:base:configure - .variables:default - .before_script:default + - .rules:nightly-only-for-release + # Built by admin/dockerfiles/ci-docs-clang + # TODO this should be organized more like the current documentation.py script + image: gromacs/ci-docs-clang:master + stage: release-configure cache: {} variables: KUBERNETES_CPU_LIMIT: 1 @@ -95,14 +90,6 @@ RELEASE_BUILD_DIR: release-doc-builds RELEASE_SOURCE: release-source-from-tarball RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball - - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE script: - if [[ $GROMACS_RELEASE == "true" ]] ; then REL_OPTION="-DGMX_BUILD_TARBALL=ON" ; @@ -189,26 +176,18 @@ docs:configure: extends: - .docs:configure + - .rules:not-for-release needs: - job: simple-build artifacts: false variables: BUILD_DIR: build-docs CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on - except: - variables: - - $GROMACS_RELEASE .docs:build: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master - only: - refs: - - web - - pushes - - schedules - - merge_requests script: - cd $BUILD_DIR - cmake --build . --target gmx -- -j8 @@ -227,6 +206,7 @@ docs:build: - .use-ccache - .docs:build - .before_script:default + - .rules:not-for-release stage: documentation needs: - job: simple-build @@ -235,9 +215,6 @@ docs:build: artifacts: true variables: BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE # The manual build is built separately so that errors in converting # Sphinx to LaTeX and compiling can always be found in the @@ -255,12 +232,6 @@ docs:build: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master - only: - refs: - - web - - pushes - - schedules - - merge_requests variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 @@ -294,6 +265,7 @@ docs:build: webpage:build: extends: - .webpage:build + - .rules:not-for-release stage: post-test needs: - job: docs:build @@ -302,21 +274,12 @@ webpage:build: artifacts: false variables: BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE webpage:configure: extends: - .docs:configure:release + - .rules:nightly-only-for-release cache: {} - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -332,33 +295,21 @@ webpage:dependencies: - .use-ccache - .docs:build - .before_script:default + - .rules:nightly-only-for-release stage: release-build needs: - job: webpage:configure artifacts: true variables: BUILD_DIR: release-doc-builds - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE webpage:build:release: extends: - .webpage:build + - .rules:nightly-only-for-release dependencies: - webpage:dependencies - webpage:configure stage: release-prepare-deploy - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE variables: BUILD_DIR: release-doc-builds diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml index 62c4471695..1ece0efb6a 100644 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -27,6 +27,41 @@ - export CCACHE_BASEDIR=${PWD} - export CCACHE_DIR=${PWD}/ccache +# Rule to run a job only in nightly release-preparation pipelines. +# Checks if the GROMACS_RELEASE variable was set (typically through the GitLab web interface). +# Excludes merge_requests +# TODO: Update to *rules* syntax. +.rules:nightly-only-for-release: + only: + refs: + - web + - schedules + variables: + - $GROMACS_RELEASE + +# Jobs that run on schedules, but not for merge requests or when GROMACS_RELEASE +# is set. +.rules:nightly-not-for-release: + only: + - web + - schedules + except: + variables: + - $GROMACS_RELEASE + +# Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE +# is set. +.rules:not-for-release: + only: + refs: + - web + - pushes + - schedules + - merge_requests + except: + variables: + - $GROMACS_RELEASE + # Behavioral templates # Use a persistent compiler cache to speed up rebuilds for a single job. diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index b9ecb75289..7a083a3170 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -6,6 +6,7 @@ simple-build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:not-for-release stage: pre-build variables: KUBERNETES_CPU_LIMIT: 8 @@ -20,15 +21,6 @@ simple-build: CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" BUILD_DIR: simple-build - only: - refs: - - web - - pushes - - schedules - - merge_requests - except: - variables: - - $GROMACS_RELEASE script: - echo $CMAKE_COMPILER_SCRIPT - echo $CMAKE_EXTRA_OPTIONS @@ -91,12 +83,6 @@ simple-build: - .variables:default cache: {} stage: configure-build - only: - refs: - - web - - pushes - - schedules - - merge_requests variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -163,13 +149,6 @@ simple-build: RELEASE_BUILD_DIR: release-builds RELEASE_SOURCE: release-source-from-tarball RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE script: - VERSION=`cat version.json | python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` @@ -240,27 +219,19 @@ gromacs:gcc-7:configure: extends: - .gromacs:base:configure - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7:release:configure: extends: - .gromacs:base:release:configure - .use-gcc7 + - .rules:nightly-only-for-release cache: {} variables: RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -270,43 +241,33 @@ gromacs:gcc-7-simd-sse41:configure: extends: - .gromacs:base:configure - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false variables: CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:configure: extends: - .gromacs:base:configure - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false variables: CMAKE_PRECISION_OPTIONS: -DGMX_DOUBLE=ON - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:release:configure: extends: - .gromacs:base:release:configure - .use-gcc7 + - .rules:nightly-only-for-release cache: {} variables: CMAKE_PRECISION_OPTIONS: -DGMX_DOUBLE=ON RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -316,27 +277,19 @@ gromacs:gcc-8:configure: extends: - .gromacs:base:configure - .use-gcc8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-8:release:configure: extends: - .gromacs:base:release:configure - .use-gcc8 + - .rules:nightly-only-for-release cache: {} variables: RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -347,15 +300,13 @@ gromacs:gcc-8-cuda-10.2:configure: - .gromacs:base:configure - .use-cuda - .use-gcc8:cuda-10.2 + - .rules:not-for-release variables: CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE # For version of clang before 7, we need to always provide the full version # Use something like this instead in the download script: @@ -364,41 +315,31 @@ gromacs:clang-6:configure: extends: - .gromacs:base:configure - .use-clang6 + - .rules:not-for-release needs: - job: simple-build artifacts: false variables: COMPILER_MAJOR_VERSION: "6.0" CMAKE_MPI_OPTIONS: -DGMX_OPENMP=OFF - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:configure: extends: - .gromacs:base:configure - .use-clang7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:release:configure: extends: - .gromacs:base:release:configure - .use-clang7 + - .rules:nightly-only-for-release cache: {} variables: RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -408,27 +349,19 @@ gromacs:clang-8:configure: extends: - .gromacs:base:configure - .use-clang8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - except: - variables: - - $GROMACS_RELEASE gromacs:clang-8:release:configure: extends: - .gromacs:base:release:configure - .use-clang8 + - .rules:nightly-only-for-release cache: {} variables: RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE dependencies: - archive:package - regressiontests:package @@ -440,12 +373,6 @@ gromacs:clang-8:release:configure: # Dockerfiles are from dockerhub, user eriklindahl # image in admin/dockerimages/ci-docs-py27 stage: build - only: - refs: - - web - - pushes - - schedules - - merge_requests script: - cd $BUILD_DIR - cmake --build . -- -j8 2>&1 | tee buildLogFile.log @@ -468,14 +395,12 @@ gromacs:gcc-7:build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-7:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7:release:build: extends: @@ -483,19 +408,13 @@ gromacs:gcc-7:release:build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:nightly-only-for-release stage: release-build variables: BUILD_DIR: release-builds-gcc needs: - job: gromacs:gcc-7:release:configure artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE gromacs:gcc-7-simd-sse41:build: extends: @@ -503,14 +422,12 @@ gromacs:gcc-7-simd-sse41:build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-7-simd-sse41:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:build: extends: @@ -518,14 +435,12 @@ gromacs:gcc-7-double:build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-7-double:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:release:build: extends: @@ -533,19 +448,13 @@ gromacs:gcc-7-double:release:build: - .use-ccache - .variables:default - .use-gcc7 + - .rules:nightly-only-for-release stage: release-build variables: BUILD_DIR: release-builds-gcc needs: - job: gromacs:gcc-7-double:release:configure artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE gromacs:gcc-8:build: extends: @@ -553,14 +462,12 @@ gromacs:gcc-8:build: - .use-ccache - .variables:default - .use-gcc8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-8:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-8:release:build: extends: @@ -568,19 +475,13 @@ gromacs:gcc-8:release:build: - .use-ccache - .variables:default - .use-gcc8 + - .rules:nightly-only-for-release stage: release-build variables: BUILD_DIR: release-builds-gcc needs: - job: gromacs:gcc-8:release:configure artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE gromacs:gcc-8-cuda-10.2:build: extends: @@ -589,6 +490,7 @@ gromacs:gcc-8-cuda-10.2:build: - .variables:default - .use-cuda - .use-gcc8:cuda-10.2 + - .rules:not-for-release variables: CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. @@ -597,9 +499,6 @@ gromacs:gcc-8-cuda-10.2:build: artifacts: false - job: gromacs:gcc-8-cuda-10.2:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:clang-6:build: extends: @@ -607,14 +506,12 @@ gromacs:clang-6:build: - .use-ccache - .variables:default - .use-clang6 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:clang-6:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:build: extends: @@ -622,14 +519,12 @@ gromacs:clang-7:build: - .use-ccache - .variables:default - .use-clang7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:clang-7:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:release:build: extends: @@ -637,19 +532,13 @@ gromacs:clang-7:release:build: - .use-ccache - .variables:default - .use-clang7 + - .rules:nightly-only-for-release stage: release-build variables: BUILD_DIR: release-builds-clang needs: - job: gromacs:clang-7:release:configure artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE gromacs:clang-8:build: extends: @@ -657,14 +546,12 @@ gromacs:clang-8:build: - .use-ccache - .variables:default - .use-clang8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:clang-8:configure artifacts: true - except: - variables: - - $GROMACS_RELEASE gromacs:clang-8:release:build: extends: @@ -672,24 +559,19 @@ gromacs:clang-8:release:build: - .use-ccache - .variables:default - .use-clang8 + - .rules:nightly-only-for-release stage: release-build variables: BUILD_DIR: release-builds-clang needs: - job: gromacs:clang-8:release:configure artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - # Jobs running during test stage .gromacs:base:test: + extends: + - .variables:default variables: REGRESSIONTEST_TOTAL_RANK_NUMBER: 4 REGRESSIONTEST_OMP_RANK_NUMBER: 2 @@ -699,14 +581,6 @@ gromacs:clang-8:release:build: stage: test retry: max: 2 - extends: - - .variables:default - only: - refs: - - web - - pushes - - schedules - - merge_requests script: - cd $BUILD_DIR - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true @@ -737,19 +611,18 @@ gromacs:gcc-7:test: extends: - .gromacs:base:test - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-7:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7:release:test: extends: - .gromacs:base:test - .use-gcc7 + - .rules:nightly-only-for-release stage: release-tests variables: BUILD_DIR: release-builds-gcc @@ -758,31 +631,23 @@ gromacs:gcc-7:release:test: artifacts: true - job: gromacs:gcc-7:release:build artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE gromacs:gcc-7-simd-sse41:test: extends: - .gromacs:base:test - .use-gcc7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-7-simd-sse41:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:test: extends: - .gromacs:base:test - .use-gcc7 + - .rules:not-for-release variables: REGRESSIONTEST_DOUBLE: "-double" needs: @@ -790,14 +655,12 @@ gromacs:gcc-7-double:test: artifacts: false - job: gromacs:gcc-7-double:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-7-double:release:test: extends: - .gromacs:base:test - .use-gcc7 + - .rules:nightly-only-for-release stage: release-tests variables: BUILD_DIR: release-builds-gcc @@ -806,31 +669,23 @@ gromacs:gcc-7-double:release:test: artifacts: true - job: gromacs:gcc-7-double:release:build artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE gromacs:gcc-8:test: extends: - .gromacs:base:test - .use-gcc8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:gcc-8:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:gcc-8:release:test: extends: - .gromacs:base:test - .use-gcc8 + - .rules:nightly-only-for-release stage: release-tests variables: BUILD_DIR: release-builds-gcc @@ -839,15 +694,13 @@ gromacs:gcc-8:release:test: artifacts: true - job: gromacs:gcc-8:release:build artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE gromacs:gcc-8-cuda-10.2:test: + extends: + - .gromacs:base:test + - .use-cuda + - .use-gcc8:cuda-10.2 + - .rules:not-for-release variables: KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu" KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 @@ -856,23 +709,17 @@ gromacs:gcc-8-cuda-10.2:test: REGRESSIONTEST_OMP_RANK_NUMBER: 1 CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. - extends: - - .gromacs:base:test - - .use-cuda - - .use-gcc8:cuda-10.2 needs: - job: simple-build artifacts: false - job: gromacs:gcc-8-cuda-10.2:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:clang-6:test: extends: - .gromacs:base:test - .use-clang6 + - .rules:not-for-release variables: REGRESSIONTEST_OMP_RANK_NUMBER: 0 needs: @@ -880,27 +727,23 @@ gromacs:clang-6:test: artifacts: false - job: gromacs:clang-6:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:test: extends: - .gromacs:base:test - .use-clang7 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:clang-7:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:clang-7:release:test: extends: - .gromacs:base:test - .use-clang7 + - .rules:nightly-only-for-release stage: release-tests variables: BUILD_DIR: release-builds-clang @@ -909,31 +752,23 @@ gromacs:clang-7:release:test: artifacts: true - job: gromacs:clang-7:release:build artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE gromacs:clang-8:test: extends: - .gromacs:base:test - .use-clang8 + - .rules:not-for-release needs: - job: simple-build artifacts: false - job: gromacs:clang-8:build - job: regressiontests:prepare - except: - variables: - - $GROMACS_RELEASE gromacs:clang-8:release:test: extends: - .gromacs:base:test - .use-clang8 + - .rules:nightly-only-for-release stage: release-tests variables: BUILD_DIR: release-builds-clang @@ -942,10 +777,3 @@ gromacs:clang-8:release:test: artifacts: true - job: gromacs:clang-8:release:build artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index 19ce0c7c38..6fcdf2e2d5 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -4,6 +4,7 @@ clang-tidy:configure: extends: - .gromacs:base:configure - .use-clang7 + - .rules:not-for-release needs: - job: simple-build artifacts: false @@ -13,9 +14,6 @@ clang-tidy:configure: EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" # TODO: Enable GMX_PYTHON_PACKAGE for clang-tidy when dependencies are in Docker image. - except: - variables: - - $GROMACS_RELEASE clang-tidy:build: extends: @@ -23,6 +21,7 @@ clang-tidy:build: - .use-ccache - .variables:default - .use-clang7 + - .rules:not-for-release stage: source-check needs: - job: simple-build @@ -32,15 +31,13 @@ clang-tidy:build: variables: BUILD_DIR: build-clang-tidy EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION - except: - variables: - - $GROMACS_RELEASE clang-format: extends: - .gromacs:base:build - .variables:default - .use-clang7 + - .rules:not-for-release cache: {} stage: source-check when: always @@ -52,9 +49,6 @@ clang-format: KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi EXTRA_INSTALLS: clang-format-$COMPILER_MAJOR_VERSION - except: - variables: - - $GROMACS_RELEASE script: - export CLANG_FORMAT=clang-format-$COMPILER_MAJOR_VERSION - admin/clang-format.sh check --rev=HEAD^ --warnings=clang-format.log @@ -71,6 +65,7 @@ copyright-check: - .gromacs:base:build - .variables:default - .use-clang7 + - .rules:not-for-release cache: {} stage: source-check when: always @@ -81,9 +76,6 @@ copyright-check: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi - except: - variables: - - $GROMACS_RELEASE script: - admin/copyright.sh check --rev=HEAD^ --warnings=copyright.log - grep -iq "copyright year" copyright.log | tee years.log || true @@ -104,6 +96,7 @@ check-source: - .variables:default - .docs:build - .before_script:default + - .rules:not-for-release cache: {} stage: source-check needs: @@ -116,9 +109,6 @@ check-source: KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE script: - cd $BUILD_DIR - cmake --build . --target check-source @@ -136,17 +126,10 @@ check-source: linkchecker: extends: - .webpage:build + - .rules:nightly-not-for-release stage: nightly-build dependencies: - webpage:build - only: - - web - - schedules - - triggers - - merge_requests - except: - variables: - - $GROMACS_RELEASE variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst index 946a9af2e1..7b51a21dd0 100644 --- a/docs/dev-manual/gitlab.rst +++ b/docs/dev-manual/gitlab.rst @@ -67,11 +67,44 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions. directly, *extend* a *.use_* template from :file:`admin/gitlab-ci/global.gitlab-ci.yml` + rules + only + except + when + *Job* parameters for controlling the circumstances under which jobs run. + (Some key words may have different meanings when occurring as elements + of other parameters, such as *archive:when*, to which this note is not + intended to apply.) + Instead of setting any of these directly in a job definition, try to use + one of the pre-defined behaviors (defined as ``.rules:`` in + :file:`admin/gitlab-ci/global.gitlab-ci.yml`). + Errors or unexpected behavior will occur if you specify more than one + *.rules:...* template, or if you use these parameters in combination + with a *.rules...* template. + To reduce errors and unexpected behavior, restrict usage of these controls + to regular job definitions (don't use in "hidden" or parent jobs). + variables Many job definitions will add or override keys in *variables*. Refer to `GitLab `__ for details of the merging behavior. Refer to :ref:`variables` for local usage. +Schedules and triggers +~~~~~~~~~~~~~~~~~~~~~~ + +Pipeline `schedules `__ are +configured through the GitLab web interface. +Scheduled pipelines may provide different variable definitions through the +environment to jobs that run under the ``schedules`` +`condition `__. + +Nightly scheduled pipelines run against ``master`` and *release* branches in +the GROMACS repository. + +Pending resolution of :issue:`3272`, +the repository hosted on Gerrit is mirrored to GitLab. +Mirror updates trigger pipeline builds. + Global templates ~~~~~~~~~~~~~~~~ @@ -142,6 +175,12 @@ Other important variable keys are as follows. CMAKE_MPI_OPTIONS Provide CMake command line arguments to define GROMACS MPI build options. + GROMACS_RELEASE + Read-only environment variable that can be checked to see if a job is + executing in a pipeline for preparing a tagged release. + Can be set when launching pipelines via the GitLab web interface. + For example, see *rules* mix-ins in :file:`admin/gitlab-ci/global.gitlab-ci.yml`. + EXTRA_INSTALLS List additional OS package requirements. Used in *before_script* for some mix-in job definitions to install additional software dependencies. If -- 2.22.0