Add required Infrastructure for Gitlab merges

Also adds runs for regressiontests to normal CI.

Change-Id: I7bcb62cd3f2120f1d2e80b2f8a2ec1e328626902
(cherry picked from commit f518d72b2bea9a0dba02695dc93ca55ef1b09139)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 337cac99ec081bd0f2f798f630c02cffcce82527..9afda30b5d0c278ad466e3fb5f1918dae46a587f 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ stages:
   - configure-build
   # Build GROMACS in multiple configurations
   - build
-  # Test each configuration, perhaps in multiple ways
+  # Unit test each configuration, perhaps in multiple ways
   - test
   # Generate documentation files
   - documentation

diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml
index ef75cd992608b0c2370770fda421acd77cdb8a94..e1cc6cea431233d512f65247209444c0e06de8ca 100644 (file)
--- a/admin/gitlab-ci/archive.gitlab-ci.yml
+++ b/admin/gitlab-ci/archive.gitlab-ci.yml
@@ -27,7 +27,7 @@
 
 prepare-release-version:
   extends:
-    - .regressiontests-extends-template
+    - .regressiontests-prepare-extends-template
     - .release-version-template
 
 # Special job to package regressiontest files and have them available for testing
@@ -43,34 +43,31 @@ prepare-release-version:
     KUBERNETES_MEMORY_LIMIT: 2Gi
   only:
     refs:
+      - web
+      - pushes
       - merge_requests
       - schedules
-      - web
   except:
     variables:
       - $GROMACS_RELEASE
   # Always clone the default version for this branch, master in this case
   script:
     - export REGTESTBRANCH=master
-    - if [ ! -z $CI_MERGE_REQUEST_TARGET_BRANCH_NAME ] ; then
-      if [[ ! -d regressiontests ]] ; then
-      mkdir regressiontests ;
-      cd regressiontests ;
-      git init ;
-      cd .. ;
-      fi ;
-      cd regressiontests ;
-      git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH ;
-      git checkout -qf FETCH_HEAD ;
-      git clean -ffdxq ;
-      git gc ;
-      git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD;
-      echo "Build regressiontests for branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME";
-      cd ..;
-      else
-      touch regressiontests;
-      tar czf gmx-regressiontests.tar.gz regressiontests ;
-      fi;
+    - if [[ ! -d regressiontests ]] ; then
+        mkdir regressiontests ;
+        cd regressiontests ;
+        git init ;
+        cd .. ;
+      fi
+    - cd regressiontests
+    - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
+    - git checkout -qf FETCH_HEAD
+    - git clean -ffdxq
+    - git gc
+    - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
+    - echo "Build regressiontests for branch $REGTESTBRANCH"
+    - mv gmx-regressiontests.tar.gz ..
+    - cd ..
   artifacts:
     paths:
       - gmx-regressiontests.tar.gz
@@ -87,8 +84,8 @@ prepare-release-version:
     KUBERNETES_MEMORY_LIMIT: 2Gi
   only:
     refs:
-      - schedules
       - web
+      - schedules
       - triggers
     variables:
       - $GROMACS_RELEASE
@@ -122,19 +119,19 @@ prepare-release-version:
     paths:
       - regressiontests-*.tar.gz
 
-.regressiontests-extends-template:
+.regressiontests-prepare-extends-template:
   extends:
     - .variables:default
   cache: {}
 
 prepare-regressiontests:
   extends:
-    - .regressiontests-extends-template
+    - .regressiontests-prepare-extends-template
     - .regressiontests-template
 
 package-regressiontests-release:
   extends:
-    - .regressiontests-extends-template
+    - .regressiontests-prepare-extends-template
     - .regressiontests-release-template
   needs:
     - job: prepare-release-version
@@ -150,6 +147,7 @@ package-regressiontests-release:
 configure-archive-nightly:
   stage: nightly-configure
   only:
+    - web
     - triggers
     - schedules
   needs:
@@ -191,6 +189,12 @@ configure-archive-release:
 .archive-build-template:
   # Dockerfiles are from dockerhub, user eriklindahl
   image: gromacs/ci-docs-clang:master
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   variables:
     KUBERNETES_CPU_LIMIT: 4
     KUBERNETES_CPU_REQUEST: 2
@@ -251,6 +255,7 @@ build-archive-nightly:
     - job: configure-archive-nightly
       artifacts: true
   only:
+    - web
     - triggers
     - schedules
   except:
@@ -316,7 +321,7 @@ package-archive-release:
 
 release-verify:
   extends:
-    - .regressiontests-extends-template
+    - .regressiontests-prepare-extends-template
     - .release-verify-template
   only:
     refs:

diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml
index d808a57c8d277e6fa81452280f37adeb04a5c5f6..2731ef36fed7e1e87362b5d0e213b8533e5686e0 100644 (file)
--- a/admin/gitlab-ci/documentation.gitlab-ci.yml
+++ b/admin/gitlab-ci/documentation.gitlab-ci.yml
@@ -17,6 +17,12 @@
   # TODO this should be organized more like the current documentation.py script
   image: gromacs/ci-docs-clang:master
   stage: configure-build
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
@@ -29,7 +35,6 @@
     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
-
   script:
     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
@@ -230,6 +235,12 @@ configure-docs-gmxapi:
   # Built by admin/dockerfiles/ci-docs-clang
   # TODO this should be organized more like the current documentation.py script
   image: gromacs/ci-docs-clang:master
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   script:
     - cd $BUILD_DIR
     - cmake --build . --target gmx -- -j8
@@ -245,6 +256,12 @@ configure-docs-gmxapi:
   # Built by admin/dockerfiles/ci-docs-clang
   # TODO this should be organized more like the current documentation.py script
   image: gromacs/ci-docs-clang:master
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   variables:
     KUBERNETES_CPU_LIMIT: 4
     KUBERNETES_CPU_REQUEST: 2
@@ -368,6 +385,7 @@ webpage-nightly:
     - .configure-gmxapi-template
   stage: nightly-deploy
   only:
+    - web
     - triggers
     - schedules
   except:

diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml
index d0afdf10775ff723064547539ef960618a326fc3..c4017601a8b4888ad6582f59ac394b9b063cb4bb 100644 (file)
--- a/admin/gitlab-ci/gromacs.gitlab-ci.yml
+++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml
@@ -20,6 +20,12 @@ simple-build:
     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
     BUILD_DIR: simple-build
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -85,6 +91,12 @@ simple-build:
     - .variables:default
   cache: {}
   stage: configure-build
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
@@ -491,6 +503,12 @@ gromacs:clang-8:release:configure:
   # Dockerfiles are from dockerhub, user eriklindahl
   # image in admin/dockerimages/ci-docs-py27
   stage: build
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   script:
     - cd $BUILD_DIR
     - cmake --build . -- -j8 2>&1 | tee buildLogFile.log
@@ -806,11 +824,23 @@ gromacs:clang-8:release:build:
 # Jobs running during test stage
 
 .gromacs:base:test:
+  variables:
+    REGRESSIONTEST_TOTAL_RANK_NUMBER: 4
+    REGRESSIONTEST_OMP_RANK_NUMBER: 2
+    REGRESSIONTEST_PME_RANK_NUMBER: 1
+    REGRESSIONTEST_MPI_RUN_COMMAND: ""
+    REGRESSIONTEST_DOUBLE: ""
   stage: test
   retry:
     max: 2
   extends:
     - .variables:default
+  only:
+    refs:
+      - web
+      - pushes
+      - schedules
+      - merge_requests
   script:
     - cd $BUILD_DIR
     - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true
@@ -822,12 +852,18 @@ gromacs:clang-8:release:build:
       exit 1;
       fi
     - cd ..
+    - tar xzf gmx-regressiontests.tar.gz
+    - source $INSTALL_DIR/bin/GMXRC
+    - cd regressiontests
+    - echo "perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND"
+    - perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND -xml all
   artifacts:
     reports:
       junit: $BUILD_DIR/JUnitTestResults.xml
     paths:
       - $BUILD_DIR/Testing
       - $BUILD_DIR/tests
+      - regressiontests
     when: always
     expire_in: 1 week
 
@@ -839,6 +875,7 @@ gromacs:gcc-7:test:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-7:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -871,6 +908,7 @@ gromacs:gcc-7-simd-sse41:test:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-7-simd-sse41:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -879,10 +917,13 @@ gromacs:gcc-7-double:test:
   extends:
     - .gromacs:base:test
     - .use-gcc7
+  variables:
+    REGRESSIONTEST_DOUBLE: "-double"
   needs:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-7-double:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -916,6 +957,7 @@ gromacs:gcc-7:test-gmxapi:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-7:build-gmxapi
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -949,6 +991,7 @@ gromacs:gcc-8:test:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-8:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -981,10 +1024,15 @@ gromacs:gcc-8-cuda-10.2:test:
     - .gromacs:base:test
     - .use-cuda
     - .use-gcc8:cuda-10.2
+  variables:
+    REGRESSIONTEST_PME_RANK_NUMBER: 0
+    REGRESSIONTEST_TOTAL_RANK_NUMBER: 2
+    REGRESSIONTEST_OMP_RANK_NUMBER: 1
   needs:
     - job: simple-build
       artifacts: false
     - job: gromacs:gcc-8-cuda-10.2:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -993,10 +1041,13 @@ gromacs:clang-6:test:
   extends:
     - .gromacs:base:test
     - .use-clang6
+  variables:
+    REGRESSIONTEST_OMP_RANK_NUMBER: 0
   needs:
     - job: simple-build
       artifacts: false
     - job: gromacs:clang-6:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -1009,6 +1060,7 @@ gromacs:clang-7:test:
     - job: simple-build
       artifacts: false
     - job: gromacs:clang-7:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -1042,6 +1094,7 @@ gromacs:clang-7:test-gmxapi:
     - job: simple-build
       artifacts: false
     - job: gromacs:clang-7:build-gmxapi
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE
@@ -1075,6 +1128,7 @@ gromacs:clang-8:test:
     - job: simple-build
       artifacts: false
     - job: gromacs:clang-8:build
+    - job: prepare-regressiontests
   except:
     variables:
       - $GROMACS_RELEASE

diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml
index b0ea30edc7abf3c371cf1e8adac2bfa6c94fe16d..361d97c9f828bad5e4f0c75b3358a0277dc36358 100644 (file)
--- a/admin/gitlab-ci/lint.gitlab-ci.yml
+++ b/admin/gitlab-ci/lint.gitlab-ci.yml
@@ -138,10 +138,10 @@ linkchecker:
   dependencies:
     - webpage-gmxapi
   only:
+    - web
     - schedules
     - triggers
     - merge_requests
-    - external_pull_requests
   except:
     variables:
       - $GROMACS_RELEASE