Use CMake to propagate versions and hashes to gitlab jobs
[alexxy/gromacs.git] / admin / gitlab-ci / archive.gitlab-ci.yml
index 1d009808ee0644ddc8e940b56e88ab8756dc4ab6..527c90a58876e5899c07690fcabcdb86804fd84b 100644 (file)
@@ -1,25 +1,24 @@
 # Packages, exported artifacts, and release engineering processes.
 
+# Special job to run cmake to output the version string and
+# regressiontest md5sum to a file
+
 prepare-release-version:
   extends:
     - .variables:default
     - .rules:nightly-only-for-release
   cache: {}
-  # Docker image uploaded to dockerhub by user eriklindahl
-  # TODO: Get DockerFile for admin/dockerfiles
-  image: biophysics/gcc-gromacs
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: configure-build
-
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
   script:
-    - cmake -P cmake/gmxVersionInfo.cmake &> version.json
-
+    - cmake . -DGMX_BUILD_TARBALL=on
   artifacts:
     paths:
-      - version.json
+      - VersionInfo-partial.cmake.cmakein
 
 # Special job to package regressiontest files and have them available for testing
 # Runs during pre-build
@@ -27,18 +26,26 @@ prepare-release-version:
 regressiontests:prepare:
   extends:
     - .variables:default
-    - .rules:not-for-release
+    - .rules:merge-and-post-merge-acceptance
   cache: {}
-  image: biophysics/gcc-gromacs
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: configure-build
-
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
   # Always clone the default version for this branch, master in this case
   script:
     - export REGTESTBRANCH=master
+    - if [[ ! -z $REGRESSIONTESTBRANCH ]] ; then
+      export REGTESTBRANCH=$REGRESSIONTESTBRANCH ;
+      echo "Using $REGTESTBRANCH instead of default" ;
+      fi
+    - export CHECKOUTCOMMIT=FETCH_HEAD
+    - if [[ ! -z $REGRESSIONTESTCOMMIT ]] ; then
+      export CHECKOUTCOMMIT=$REGRESSIONTESTCOMMIT ;
+      echo "Using $CHECKOUTCOMMIT instead of head of $REGTESTBRANCH" ;
+      fi
     - if [[ ! -d regressiontests ]] ; then
         mkdir regressiontests ;
         cd regressiontests ;
@@ -47,7 +54,7 @@ regressiontests:prepare:
       fi
     - cd regressiontests
     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
-    - git checkout -qf FETCH_HEAD
+    - git checkout -qf $CHECKOUTCOMMIT
     - git clean -ffdxq
     - git gc
     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
@@ -65,19 +72,16 @@ regressiontests:package:
   cache: {}
   # Docker image uploaded to dockerhub by user eriklindahl
   # TODO: Get DockerFile for admin/dockerfiles
-  image: biophysics/gcc-gromacs
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: release-package
-
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
   needs:
     - job: prepare-release-version
-      artifacts: true
   script:
-    - VERSION=`cat version.json |
-      python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
+    - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
     - REGTESTNAME="regressiontests-"$VERSION
     - if [[ $GROMACS_RELEASE != "true" ]] ; then
       REGTESTNAME=$REGTESTNAME-dev ;
@@ -98,9 +102,9 @@ regressiontests:package:
     - git clean -ffdxq
     - git gc
     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
-    - cp $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz
+    - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
-    - echo "Build regressiontests for branch  $REGTESTBRANCH"
+    - echo "Build regressiontests $REGTESTNAME for branch  $REGTESTBRANCH"
     - cd ..
   artifacts:
     paths:
@@ -116,7 +120,7 @@ archive:configure:nightly:
   variables:
     BUILD_DIR: build-package
     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
-    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
+    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 
 archive:configure:release:
@@ -126,7 +130,7 @@ archive:configure:release:
   variables:
     BUILD_DIR: build-package
     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
-    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
+    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 
 # Right now we need to use an ugly hack to move the original build directory
@@ -136,17 +140,16 @@ archive:configure:release:
 # manually hacking things. TODO: #3361
 .archive:build:
   extends:
+    - .variables:default
     - .gromacs:base:build
     - .use-ccache
-    - .variables:default
     - .before_script:default
     - .docs:build
-  # Dockerfiles are from dockerhub, user eriklindahl
-  image: gromacs/ci-docs-clang:master
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   variables:
     KUBERNETES_CPU_LIMIT: 4
     KUBERNETES_CPU_REQUEST: 2
-    KUBERNETES_MEMORY_LIMIT: 4Gi
+    KUBERNETES_MEMORY_REQUEST: 4Gi
   script:
     - rm -rf gromacs*tar.gz
     - cd $BUILD_DIR
@@ -181,7 +184,9 @@ archive:configure:release:
     when: always
     expire_in: 1 week
     paths:
-      - $BUILD_DIR/*log
+      - $BUILD_DIR/*logs
+      - $BUILD_DIR/bin
+      - $BUILD_DIR/lib
       - gromacs*tar.gz
 
 archive:build:
@@ -191,7 +196,6 @@ archive:build:
   stage: nightly-build
   needs:
     - job: archive:configure:nightly
-      artifacts: true
   variables:
     BUILD_DIR: build-package
 
@@ -202,12 +206,11 @@ archive:package:
   stage: release-package
   needs:
     - job: archive:configure:release
-      artifacts: true
   variables:
     BUILD_DIR: build-package
 
-release-verify:
-  image: biophysics/gcc-gromacs
+checksum-verify:
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: release-verify
   extends:
     - .variables:default
@@ -217,26 +220,29 @@ release-verify:
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
+    BUILD_DIR: build-package
   script:
-    - VERSION=`cat version.json |
-      python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
-    - if [[ $GROMACS_RELEASE != "true" ]] ; then
-      VERSION=$VERSION-dev ;
+    - VALID_BUILD=true
+    - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
+    - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
+      VERSION="$VERSION"-dev ;
       fi
-    - REGTEST_COMPARE=`cat version.json |
-      python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
-    - SOURCENAME=gromacs-$VERSION
-    - SOURCETARBALL=$SOURCENAME.tar.gz
-    - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
-    - REGTESTNAME=regressiontests-$VERSION
-    - REGTESTTARBALL=$REGTESTNAME.tar.gz
-    - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'`
+    - REGTEST_COMPARE=`sed -n -e '/REGRESSIONTEST_MD5SUM/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
+    - SOURCENAME=gromacs-"$VERSION"
+    - SOURCETARBALL="$SOURCENAME".tar.gz
+    - SOURCE_MD5SUM=`md5sum "$SOURCETARBALL" | awk '{print $1}'`
+    - REGTESTNAME=regressiontests-"$VERSION"
+    - REGTESTTARBALL="$REGTESTNAME".tar.gz
+    - REGTEST_MD5SUM=`md5sum "$REGTESTTARBALL" | awk '{print $1}'`
     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
-    - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then
+    - if [[ "$REGTEST_COMPARE" != "$REGTEST_MD5SUM" && "$GROMACS_RELEASE" == "true" ]] ; then
       echo "Mismatch in regressiontest md5sums";
+      VALID_BUILD=false;
+      fi
+    - if [[ "$VALID_BUILD" != "true" ]] ; then
       exit 1;
       fi
   dependencies:
@@ -244,6 +250,39 @@ release-verify:
     - regressiontests:package
     - prepare-release-version
 
+version-verify:
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
+  stage: release-verify
+  extends:
+    - .variables:default
+    - .rules:nightly-only-for-release
+  cache: {}
+
+  variables:
+    KUBERNETES_CPU_LIMIT: 1
+    KUBERNETES_CPU_REQUEST: 1
+    KUBERNETES_MEMORY_REQUEST: 2Gi
+    BUILD_DIR: release-doc-builds
+  script:
+    - VALID_BUILD=true
+    - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
+    - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
+      VERSION="$VERSION"-dev ;
+      fi
+    - VERSION_FROM_BINARY=`"${BUILD_DIR}"/bin/gmx --version | grep "GROMACS version" | awk '{print $3}'`
+    - echo "Version name from CMake = $VERSION"
+    - echo "Version name from binary = $VERSION_FROM_BINARY"
+    - if [[ "$VERSION" != "$VERSION_FROM_BINARY" && "$GROMACS_RELEASE" == "true" ]] ; then
+      echo "Version names don't match";
+      VALID_BUILD=false;
+      fi
+    - if [[ "$VALID_BUILD" != "true" ]] ; then
+      exit 1;
+      fi
+  dependencies:
+    - webpage:dependencies
+    - prepare-release-version
+
 archive:nightly-webpage:
   extends:
     - .webpage:build
@@ -253,11 +292,19 @@ archive:nightly-webpage:
     - job: linkchecker
       artifacts: false
     - job: webpage:build
-      artifacts: true
   variables:
     BUILD_DIR: build-docs
+  before_script:
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan manual.gromacs.org > ~/.ssh/known_hosts # Force overwrite the known hosts so we only have that one key in it.
+    - chmod 644 ~/.ssh/known_hosts
   script:
     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
+    - rsync --chmod=u+rwX,g+rwX,o+rX -av $BUILD_DIR/docs/html/* $BUILD_DIR/docs/html/.[a-z]* pbauer@manual.gromacs.org:/var/www/manual/nightly/
+
   artifacts:
     when: always
     paths: