From: Paul Bauer <paul.bauer.q@gmail.com>
Date: Tue, 29 Sep 2020 13:00:15 +0000 (+0000)
Subject: Add post submit rules to CI
X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=52a77ce5569d870d484c6db2f49a349eb1bf96a8;p=alexxy%2Fgromacs.git

Add post submit rules to CI

Use those for resource intensive CUDA tests

Change-Id: I9c23139656fd91a833c67dd20d76a1429e91c254
---

diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml
index b33089dafe..15877d35ae 100644
--- a/admin/gitlab-ci/archive.gitlab-ci.yml
+++ b/admin/gitlab-ci/archive.gitlab-ci.yml
@@ -25,7 +25,7 @@ prepare-release-version:
 regressiontests:prepare:
   extends:
     - .variables:default
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   cache: {}
   image: gromacs/ci-docs-llvm-master
   stage: configure-build
diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml
index 1ba6539062..04b41d7a0a 100644
--- a/admin/gitlab-ci/gromacs.gitlab-ci.yml
+++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml
@@ -221,7 +221,7 @@ gromacs:gcc-7:configure:
     - .gromacs:base:configure
     - .use-gcc:base
     - .use-opencl
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-gcc-7-amdopencl-clfft-openmpi-master
   variables:
     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AVX2_256"
@@ -233,7 +233,7 @@ gromacs:gcc-8-cuda-11.0:configure:
     - .gromacs:base:configure
     - .use-gcc:base
     - .use-cuda
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   variables:
     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1"
@@ -245,7 +245,7 @@ gromacs:gcc-8-cuda-11.0:configureMPI:
     - .use-gcc:base
     - .use-cuda
     - .use-mpi
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   variables:
     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1"
@@ -255,7 +255,7 @@ gromacs:clang-TSAN:configure:
   extends:
     - .gromacs:base:configure
     - .use-clang:base
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   variables:
     COMPILER_MAJOR_VERSION: 8
@@ -316,7 +316,7 @@ gromacs:clang-UBSAN:configure:
   extends:
     - .gromacs:base:configure
     - .use-clang:base
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   variables:
     COMPILER_MAJOR_VERSION: 8
@@ -440,7 +440,7 @@ gromacs:gcc-7:build:
     - .gromacs:base:build
     - .before_script:default
     - .use-ccache
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-gcc-7-amdopencl-clfft-openmpi-master
   needs:
     - job: gromacs:gcc-7:configure
@@ -452,7 +452,7 @@ gromacs:gcc-8-cuda-11.0:build:
     - .gromacs:base:build
     - .before_script:default
     - .use-ccache
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   needs:
     - job: gromacs:gcc-8-cuda-11.0:configure
@@ -463,7 +463,7 @@ gromacs:gcc-8-cuda-11.0:buildMPI:
     - .gromacs:base:build
     - .before_script:default
     - .use-ccache
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   needs:
     - job: gromacs:gcc-8-cuda-11.0:configureMPI
@@ -474,7 +474,7 @@ gromacs:clang-TSAN:build:
     - .gromacs:base:build
     - .use-clang:base
     - .use-ccache
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   needs:
     - job: gromacs:clang-TSAN:configure
@@ -498,7 +498,7 @@ gromacs:clang-UBSAN:build:
     - .gromacs:base:build
     - .use-clang:base
     - .use-ccache
-    - .rules:merge-requests
+    - .rules:merge-and-post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   tags:
     - k8s-scilifelab
@@ -732,7 +732,7 @@ gromacs:gcc-8-cuda-11.0:test:
 gromacs:gcc-8-cuda-11.0:test-gpucommupd:
   extends:
     - .gromacs:base:test
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   variables:
     KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu"
@@ -756,7 +756,7 @@ gromacs:clang-8:test:
 gromacs:clang-TSAN:test:
   extends:
     - .gromacs:base:test
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   needs:
     - job: gromacs:clang-TSAN:build
@@ -778,7 +778,7 @@ gromacs:clang-UBSAN:test:
   extends:
     - .gromacs:base:test
     - .use-clang:base
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   tags:
     - k8s-scilifelab
@@ -807,7 +807,7 @@ gromacs:clang-9-mpi:test:
 gromacs:gcc-7:regressiontest:
   extends:
     - .gromacs:base:regressiontest
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.13.0-gcc-7-amdopencl-clfft-openmpi-master
   variables:
     KUBERNETES_EXTENDED_RESOURCE_NAME: "amd.com/gpu"
@@ -842,7 +842,7 @@ gromacs:gcc-8-cuda-11.0:regressiontest:
 gromacs:gcc-8-cuda-11.0:regressiontest-gpucommupd-tMPI:
   extends:
     - .gromacs:base:regressiontest
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   variables:
     KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu"
@@ -867,7 +867,7 @@ gromacs:gcc-8-cuda-11.0:regressiontest-gpucommupd-tMPI:
 gromacs:gcc-8-cuda-11.0:regressiontest-gpucommupd-MPI:
   extends:
     - .gromacs:base:regressiontest
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master
   variables:
     KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu"
@@ -904,7 +904,7 @@ gromacs:clang-8:regressiontest:
 gromacs:clang-TSAN:regressiontest:
   extends:
     - .gromacs:base:regressiontest
-    - .rules:merge-requests
+    - .rules:post-merge-acceptance
   image: gromacs/cmake-3.13.0-llvm-8-tsan-master
   tags:
     - k8s-scilifelab
diff --git a/admin/gitlab-ci/rules.gitlab-ci.yml b/admin/gitlab-ci/rules.gitlab-ci.yml
index 41be14a7bc..201e329599 100644
--- a/admin/gitlab-ci/rules.gitlab-ci.yml
+++ b/admin/gitlab-ci/rules.gitlab-ci.yml
@@ -51,6 +51,12 @@
   if: '$CI_PIPELINE_SOURCE == "schedule"'
   when: always
 
+# Exclude from selective "schedule" pipelines, e.g. just those
+# that should run the post-merge-acceptance jobs.
+.rules-element:if-post-merge-acceptance-then-never: &if-post-merge-acceptance-then-never
+  if: '$POST_MERGE_ACCEPTANCE'
+  when: never
+
 # Include in pipelines triggered in the merge request process.
 .rules-element:if-mr-then-always: &if-mr-then-always
   if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
@@ -63,6 +69,19 @@
   if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
   when: never
 
+# Include job when running for merge request or when pushing to protected branch.
+.rules-element:if-post-merge-acceptance-or-mr-then-always: &if-post-merge-acceptance-or-mr-then-always
+  if: '$CI_PIPELINE_SOURCE == "merge_request_event" ||
+       ($CI_PIPELINE_SOURCE == "push" &&
+        ($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-*"))'
+  when: always
+
+# Include job only for post submit push
+.rules-element:if-post-merge-acceptance-then-always: &if-post-merge-acceptance-then-always
+  if: '$CI_PIPELINE_SOURCE == "push" &&
+        ($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-*")'
+  when: always
+
 # When composing a rule set, note that the first matching rule is applied.
 # If you want later rules to be evaluated, you must make sure that the *if*
 # clause of the earlier rules does not match. This may require inverting the
@@ -79,23 +98,47 @@
 .rules:basic-push:
   rules:
     - *if-release-then-never
+    - *if-mr-then-never
+    - *if-post-merge-acceptance-then-never
     - *if-web-then-always
     - *if-push-then-always
     - *if-schedule-then-always
+
+# Jobs to run after successful merge of a new commit.
+# Only run on GROMACS infrastructure and only when merging into
+# the master or release branches.
+.rules:post-merge-acceptance:
+  rules:
+    - *if-not-gromacs-then-never
+    - *if-release-then-never
     - *if-mr-then-never
+    - *if-post-merge-acceptance-then-always
+    - *if-web-then-always
+    - *if-schedule-then-always
 
 # Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE
-# is set. Excludes non-gromacs projects.
+# is set. Excludes non-GROMACS projects.
 # More elaborate rule sets for merge requests should be based on the same sequence.
 .rules:merge-requests:
   rules:
     - *if-not-gromacs-then-never
     - *if-release-then-never
-    - *if-web-then-always
     - *if-push-then-never
+    - *if-post-merge-acceptance-then-never
+    - *if-web-then-always
     - *if-schedule-then-always
     - *if-mr-then-always
 
+# Jobs running both in post submit and for merge requests
+# Excludes non-GROMACS projects.
+.rules:merge-and-post-merge-acceptance:
+  rules:
+    - *if-not-gromacs-then-never
+    - *if-release-then-never
+    - *if-web-then-always
+    - *if-schedule-then-always
+    - *if-post-merge-acceptance-or-mr-then-always
+
 # Jobs that run for merge requests and schedules for branch `master`,
 # but not when GROMACS_RELEASE is set.
 # Excludes non-gromacs projects.
@@ -103,6 +146,7 @@
   rules:
     - *if-not-gromacs-then-never
     - *if-release-then-never
+    - *if-post-merge-acceptance-then-never
     - *if-web-then-always
     # This rule catches "push" and other events in branches other than `master`
     # but allows merge_request_events for merge requests targeting master.
@@ -118,6 +162,7 @@
   rules:
     - *if-not-gromacs-then-never
     - *if-release-then-never
+    - *if-post-merge-acceptance-then-never
     # This next rule catches "push" and other events in branches other than `release-2020`
     # but allows merge_request_events for merge requests targeting `release-2020`.
     # This rule is before "web" so the web interface won't include jobs that can't succeed
@@ -139,6 +184,7 @@
   rules:
     - *if-not-gromacs-then-never
     - *if-not-release-then-never
+    - *if-post-merge-acceptance-then-never
     - *if-web-then-always
     - *if-schedule-then-always
 
@@ -148,5 +194,6 @@
   rules:
     - *if-not-gromacs-then-never
     - *if-release-then-never
+    - *if-post-merge-acceptance-then-never
     - *if-web-then-always
     - *if-schedule-then-always
diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst
index 948119cba2..a7d5254f18 100644
--- a/docs/dev-manual/gitlab.rst
+++ b/docs/dev-manual/gitlab.rst
@@ -121,6 +121,15 @@ environment to jobs that run under the ``schedules``
 Nightly scheduled pipelines run against ``master`` and *release* branches in
 the GROMACS repository.
 
+Running post-merge-acceptance pipelines
+"""""""""""""""""""""""""""""""""""""""
+
+The Gitlab CI for |Gromacs| runs a set of jobs by default only after a MR has been
+accepted and the resulting commit is included in the target branch if it is ``master``
+or one of the *release* branches. Those jobs can be triggered manually using the
+``POST_MERGE_ACCEPTANCE`` input variable documented below when executing a new pipeline
+through the Gitlab web interface.
+
 Global templates
 ~~~~~~~~~~~~~~~~
 
@@ -237,6 +246,12 @@ Other important variable keys are as follows.
         allow for merge requests that require updated regression tests with 
         valid CI tests.
 
+    POST_MERGE_ACCEPTANCE
+        Read-only environment variable that indicates that only jobs scheduled to
+        run after a commit has been merged into its target branch should be executed.
+        Can be set to run pipelines through the web interface or as schedules.
+        For use please see the *rules* mix-ins in :file:`admin/gitlab-ci/global.gitlab-ci.yml`.
+
 
 .. todo:: Define common variables.
     ``BUILD_DIR``, ``INSTALL_DIR``, ``CACHE_FALLBACK_KEY``, ...