From bfc4220919da3f3bc2b6a49dec519e87d65c0659 Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Sat, 21 Mar 2020 14:27:50 +0300 Subject: [PATCH] Add required Infrastructure for Gitlab merges Also adds runs for regressiontests to normal CI. Change-Id: I7bcb62cd3f2120f1d2e80b2f8a2ec1e328626902 (cherry picked from commit f518d72b2bea9a0dba02695dc93ca55ef1b09139) --- .gitlab-ci.yml | 2 +- admin/gitlab-ci/archive.gitlab-ci.yml | 57 +++++++++++---------- admin/gitlab-ci/documentation.gitlab-ci.yml | 20 +++++++- admin/gitlab-ci/gromacs.gitlab-ci.yml | 54 +++++++++++++++++++ admin/gitlab-ci/lint.gitlab-ci.yml | 2 +- 5 files changed, 106 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 337cac99ec..9afda30b5d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - configure-build # Build GROMACS in multiple configurations - build - # Test each configuration, perhaps in multiple ways + # Unit test each configuration, perhaps in multiple ways - test # Generate documentation files - documentation diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index ef75cd9926..e1cc6cea43 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -27,7 +27,7 @@ prepare-release-version: extends: - - .regressiontests-extends-template + - .regressiontests-prepare-extends-template - .release-version-template # Special job to package regressiontest files and have them available for testing @@ -43,34 +43,31 @@ prepare-release-version: KUBERNETES_MEMORY_LIMIT: 2Gi only: refs: + - web + - pushes - merge_requests - schedules - - web except: variables: - $GROMACS_RELEASE # Always clone the default version for this branch, master in this case script: - export REGTESTBRANCH=master - - if [ ! -z $CI_MERGE_REQUEST_TARGET_BRANCH_NAME ] ; then - if [[ ! -d regressiontests ]] ; then - mkdir regressiontests ; - cd regressiontests ; - git init ; - cd .. ; - fi ; - cd regressiontests ; - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH ; - git checkout -qf FETCH_HEAD ; - git clean -ffdxq ; - git gc ; - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD; - echo "Build regressiontests for branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; - cd ..; - else - touch regressiontests; - tar czf gmx-regressiontests.tar.gz regressiontests ; - fi; + - if [[ ! -d regressiontests ]] ; then + mkdir regressiontests ; + cd regressiontests ; + git init ; + cd .. ; + fi + - cd regressiontests + - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH + - git checkout -qf FETCH_HEAD + - git clean -ffdxq + - git gc + - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD + - echo "Build regressiontests for branch $REGTESTBRANCH" + - mv gmx-regressiontests.tar.gz .. + - cd .. artifacts: paths: - gmx-regressiontests.tar.gz @@ -87,8 +84,8 @@ prepare-release-version: KUBERNETES_MEMORY_LIMIT: 2Gi only: refs: - - schedules - web + - schedules - triggers variables: - $GROMACS_RELEASE @@ -122,19 +119,19 @@ prepare-release-version: paths: - regressiontests-*.tar.gz -.regressiontests-extends-template: +.regressiontests-prepare-extends-template: extends: - .variables:default cache: {} prepare-regressiontests: extends: - - .regressiontests-extends-template + - .regressiontests-prepare-extends-template - .regressiontests-template package-regressiontests-release: extends: - - .regressiontests-extends-template + - .regressiontests-prepare-extends-template - .regressiontests-release-template needs: - job: prepare-release-version @@ -150,6 +147,7 @@ package-regressiontests-release: configure-archive-nightly: stage: nightly-configure only: + - web - triggers - schedules needs: @@ -191,6 +189,12 @@ configure-archive-release: .archive-build-template: # Dockerfiles are from dockerhub, user eriklindahl image: gromacs/ci-docs-clang:master + only: + refs: + - web + - pushes + - schedules + - merge_requests variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 @@ -251,6 +255,7 @@ build-archive-nightly: - job: configure-archive-nightly artifacts: true only: + - web - triggers - schedules except: @@ -316,7 +321,7 @@ package-archive-release: release-verify: extends: - - .regressiontests-extends-template + - .regressiontests-prepare-extends-template - .release-verify-template only: refs: diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index d808a57c8d..2731ef36fe 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -17,6 +17,12 @@ # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master stage: configure-build + only: + refs: + - web + - pushes + - schedules + - merge_requests variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -29,7 +35,6 @@ CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" - script: - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then REL_OPTION="-DGMX_BUILD_TARBALL=ON" ; @@ -230,6 +235,12 @@ configure-docs-gmxapi: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master + only: + refs: + - web + - pushes + - schedules + - merge_requests script: - cd $BUILD_DIR - cmake --build . --target gmx -- -j8 @@ -245,6 +256,12 @@ configure-docs-gmxapi: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master + only: + refs: + - web + - pushes + - schedules + - merge_requests variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 @@ -368,6 +385,7 @@ webpage-nightly: - .configure-gmxapi-template stage: nightly-deploy only: + - web - triggers - schedules except: diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index d0afdf1077..c4017601a8 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -20,6 +20,12 @@ simple-build: CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" BUILD_DIR: simple-build + only: + refs: + - web + - pushes + - schedules + - merge_requests except: variables: - $GROMACS_RELEASE @@ -85,6 +91,12 @@ simple-build: - .variables:default cache: {} stage: configure-build + only: + refs: + - web + - pushes + - schedules + - merge_requests variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -491,6 +503,12 @@ gromacs:clang-8:release:configure: # Dockerfiles are from dockerhub, user eriklindahl # image in admin/dockerimages/ci-docs-py27 stage: build + only: + refs: + - web + - pushes + - schedules + - merge_requests script: - cd $BUILD_DIR - cmake --build . -- -j8 2>&1 | tee buildLogFile.log @@ -806,11 +824,23 @@ gromacs:clang-8:release:build: # Jobs running during test stage .gromacs:base:test: + variables: + REGRESSIONTEST_TOTAL_RANK_NUMBER: 4 + REGRESSIONTEST_OMP_RANK_NUMBER: 2 + REGRESSIONTEST_PME_RANK_NUMBER: 1 + REGRESSIONTEST_MPI_RUN_COMMAND: "" + REGRESSIONTEST_DOUBLE: "" stage: test retry: max: 2 extends: - .variables:default + only: + refs: + - web + - pushes + - schedules + - merge_requests script: - cd $BUILD_DIR - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true @@ -822,12 +852,18 @@ gromacs:clang-8:release:build: exit 1; fi - cd .. + - tar xzf gmx-regressiontests.tar.gz + - source $INSTALL_DIR/bin/GMXRC + - cd regressiontests + - echo "perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND" + - perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND -xml all artifacts: reports: junit: $BUILD_DIR/JUnitTestResults.xml paths: - $BUILD_DIR/Testing - $BUILD_DIR/tests + - regressiontests when: always expire_in: 1 week @@ -839,6 +875,7 @@ gromacs:gcc-7:test: - job: simple-build artifacts: false - job: gromacs:gcc-7:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -871,6 +908,7 @@ gromacs:gcc-7-simd-sse41:test: - job: simple-build artifacts: false - job: gromacs:gcc-7-simd-sse41:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -879,10 +917,13 @@ gromacs:gcc-7-double:test: extends: - .gromacs:base:test - .use-gcc7 + variables: + REGRESSIONTEST_DOUBLE: "-double" needs: - job: simple-build artifacts: false - job: gromacs:gcc-7-double:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -916,6 +957,7 @@ gromacs:gcc-7:test-gmxapi: - job: simple-build artifacts: false - job: gromacs:gcc-7:build-gmxapi + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -949,6 +991,7 @@ gromacs:gcc-8:test: - job: simple-build artifacts: false - job: gromacs:gcc-8:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -981,10 +1024,15 @@ gromacs:gcc-8-cuda-10.2:test: - .gromacs:base:test - .use-cuda - .use-gcc8:cuda-10.2 + variables: + REGRESSIONTEST_PME_RANK_NUMBER: 0 + REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 + REGRESSIONTEST_OMP_RANK_NUMBER: 1 needs: - job: simple-build artifacts: false - job: gromacs:gcc-8-cuda-10.2:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -993,10 +1041,13 @@ gromacs:clang-6:test: extends: - .gromacs:base:test - .use-clang6 + variables: + REGRESSIONTEST_OMP_RANK_NUMBER: 0 needs: - job: simple-build artifacts: false - job: gromacs:clang-6:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -1009,6 +1060,7 @@ gromacs:clang-7:test: - job: simple-build artifacts: false - job: gromacs:clang-7:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -1042,6 +1094,7 @@ gromacs:clang-7:test-gmxapi: - job: simple-build artifacts: false - job: gromacs:clang-7:build-gmxapi + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE @@ -1075,6 +1128,7 @@ gromacs:clang-8:test: - job: simple-build artifacts: false - job: gromacs:clang-8:build + - job: prepare-regressiontests except: variables: - $GROMACS_RELEASE diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index b0ea30edc7..361d97c9f8 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -138,10 +138,10 @@ linkchecker: dependencies: - webpage-gmxapi only: + - web - schedules - triggers - merge_requests - - external_pull_requests except: variables: - $GROMACS_RELEASE -- 2.22.0