admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 prepare-release-version:
   4   extends:
   5     - .variables:default
   6     - .rules:nightly-only-for-release
   7   cache: {}
   8   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   9   stage: configure-build
  10   variables:
  11     KUBERNETES_CPU_LIMIT: 1
  12     KUBERNETES_CPU_REQUEST: 1
  13     KUBERNETES_MEMORY_REQUEST: 2Gi
  14   script:
  15     - cmake -P cmake/gmxVersionInfo.cmake &> version.json
  16   artifacts:
  17     paths:
  18       - version.json
  19
  20 # Special job to package regressiontest files and have them available for testing
  21 # Runs during pre-build
  22 # Set up to only fetch the files and prepare everything for merge requests
  23 regressiontests:prepare:
  24   extends:
  25     - .variables:default
  26     - .rules:merge-and-post-merge-acceptance
  27   cache: {}
  28   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  29   stage: configure-build
  30   variables:
  31     KUBERNETES_CPU_LIMIT: 1
  32     KUBERNETES_CPU_REQUEST: 1
  33     KUBERNETES_MEMORY_REQUEST: 2Gi
  34   # Always clone the default version for this branch, master in this case
  35   script:
  36     - export REGTESTBRANCH=master
  37     - if [[ ! -z $REGRESSIONTESTBRANCH ]] ; then
  38       export REGTESTBRANCH=$REGRESSIONTESTBRANCH ;
  39       echo "Using $REGTESTBRANCH instead of default" ;
  40       fi
  41     - export CHECKOUTCOMMIT=FETCH_HEAD
  42     - if [[ ! -z $REGRESSIONTESTCOMMIT ]] ; then
  43       export CHECKOUTCOMMIT=$REGRESSIONTESTCOMMIT ;
  44       echo "Using $CHECKOUTCOMMIT instead of head of $REGTESTBRANCH" ;
  45       fi
  46     - if [[ ! -d regressiontests ]] ; then
  47         mkdir regressiontests ;
  48         cd regressiontests ;
  49         git init ;
  50         cd .. ;
  51       fi
  52     - cd regressiontests
  53     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  54     - git checkout -qf $CHECKOUTCOMMIT
  55     - git clean -ffdxq
  56     - git gc
  57     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  58     - echo "Build regressiontests for branch $REGTESTBRANCH"
  59     - mv gmx-regressiontests.tar.gz ..
  60     - cd ..
  61   artifacts:
  62     paths:
  63       - gmx-regressiontests.tar.gz
  64
  65 regressiontests:package:
  66   extends:
  67     - .variables:default
  68     - .rules:nightly-only-for-release
  69   cache: {}
  70   # Docker image uploaded to dockerhub by user eriklindahl
  71   # TODO: Get DockerFile for admin/dockerfiles
  72   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  73   stage: release-package
  74   variables:
  75     KUBERNETES_CPU_LIMIT: 1
  76     KUBERNETES_CPU_REQUEST: 1
  77     KUBERNETES_MEMORY_REQUEST: 2Gi
  78   needs:
  79     - job: prepare-release-version
  80   script:
  81     - VERSION=`cat version.json |
  82       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
  83     - REGTESTNAME="regressiontests-"$VERSION
  84     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  85       REGTESTNAME=$REGTESTNAME-dev ;
  86       fi
  87     - export REGTESTBRANCH=master
  88     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  89       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  90       fi
  91     - if [[ ! -d regressiontests ]] ; then
  92       mkdir regressiontests ;
  93       cd regressiontests ;
  94       git init ;
  95       cd .. ;
  96       fi
  97     - cd regressiontests
  98     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  99     - git checkout -qf FETCH_HEAD
 100     - git clean -ffdxq
 101     - git gc
 102     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
 103     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
 104     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
 105     - echo "Build regressiontests for branch  $REGTESTBRANCH"
 106     - cd ..
 107   artifacts:
 108     paths:
 109       - regressiontests-*.tar.gz
 110       - gmx-regressiontests.tar.gz
 111
 112 archive:configure:nightly:
 113   extends:
 114     - .docs:configure
 115     - .rules:nightly-not-for-release
 116   stage: nightly-configure
 117   dependencies: []
 118   variables:
 119     BUILD_DIR: build-package
 120     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 121     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 122     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 123
 124 archive:configure:release:
 125   extends:
 126     - .docs:configure
 127     - .rules:nightly-only-for-release
 128   variables:
 129     BUILD_DIR: build-package
 130     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 131     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 132     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 133
 134 # Right now we need to use an ugly hack to move the original build directory
 135 # and the ccache files out of the build archive, as it is not possible to use
 136 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 137 # As soon as this is resolved we should use the separate build directories instead of
 138 # manually hacking things. TODO: #3361
 139 .archive:build:
 140   extends:
 141     - .variables:default
 142     - .gromacs:base:build
 143     - .use-ccache
 144     - .before_script:default
 145     - .docs:build
 146   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 147   variables:
 148     KUBERNETES_CPU_LIMIT: 4
 149     KUBERNETES_CPU_REQUEST: 2
 150     KUBERNETES_MEMORY_REQUEST: 4Gi
 151   script:
 152     - rm -rf gromacs*tar.gz
 153     - cd $BUILD_DIR
 154     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 155     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 156     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 157     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 158     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 159     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 160     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 161       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 162       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 163     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 164       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 165       packageBuildLogFile.log | tee -a buildErrors.log || true
 166     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 167     - mv gromacs*tar.gz ..
 168     - cd ..
 169     # ugly hack begins here
 170     - mkdir tempdir
 171     - SOURCETARBALL=`ls gromacs*tar.gz`
 172     - mv $SOURCETARBALL tempdir
 173     - cd tempdir
 174     - tar -xf $SOURCETARBALL
 175     - rm -rf gromacs-*/$BUILD_DIR
 176     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 177     - mv $SOURCETARBALL ..
 178     - cd ..
 179     - rm -rf tempdir
 180     # Hack ends here
 181   artifacts:
 182     when: always
 183     expire_in: 1 week
 184     paths:
 185       - $BUILD_DIR/*logs
 186       - $BUILD_DIR/bin
 187       - $BUILD_DIR/lib
 188       - gromacs*tar.gz
 189
 190 archive:build:
 191   extends:
 192     - .archive:build
 193     - .rules:nightly-not-for-release
 194   stage: nightly-build
 195   needs:
 196     - job: archive:configure:nightly
 197   variables:
 198     BUILD_DIR: build-package
 199
 200 archive:package:
 201   extends:
 202     - .archive:build
 203     - .rules:nightly-only-for-release
 204   stage: release-package
 205   needs:
 206     - job: archive:configure:release
 207   variables:
 208     BUILD_DIR: build-package
 209
 210 checksum-verify:
 211   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 212   stage: release-verify
 213   extends:
 214     - .variables:default
 215     - .rules:nightly-only-for-release
 216   cache: {}
 217
 218   variables:
 219     KUBERNETES_CPU_LIMIT: 1
 220     KUBERNETES_CPU_REQUEST: 1
 221     KUBERNETES_MEMORY_REQUEST: 2Gi
 222     BUILD_DIR: build-package
 223   script:
 224     - VALID_BUILD=true
 225     - VERSION=`cat version.json |
 226       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 227     - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
 228       VERSION="$VERSION"-dev ;
 229       fi
 230     - REGTEST_COMPARE=`cat version.json |
 231       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 232     - SOURCENAME=gromacs-"$VERSION"
 233     - SOURCETARBALL="$SOURCENAME".tar.gz
 234     - SOURCE_MD5SUM=`md5sum "$SOURCETARBALL" | awk '{print $1}'`
 235     - REGTESTNAME=regressiontests-"$VERSION"
 236     - REGTESTTARBALL="$REGTESTNAME".tar.gz
 237     - REGTEST_MD5SUM=`md5sum "$REGTESTTARBALL" | awk '{print $1}'`
 238     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 239     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 240     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 241     - if [[ "$REGTEST_COMPARE" != "$REGTEST_MD5SUM" && "$GROMACS_RELEASE" == "true" ]] ; then
 242       echo "Mismatch in regressiontest md5sums";
 243       VALID_BUILD=false;
 244       fi
 245     - if [[ "$VALID_BUILD" != "true" ]] ; then
 246       exit 1;
 247       fi
 248   dependencies:
 249     - archive:package
 250     - regressiontests:package
 251     - prepare-release-version
 252
 253 version-verify:
 254   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 255   stage: release-verify
 256   extends:
 257     - .variables:default
 258     - .rules:nightly-only-for-release
 259   cache: {}
 260
 261   variables:
 262     KUBERNETES_CPU_LIMIT: 1
 263     KUBERNETES_CPU_REQUEST: 1
 264     KUBERNETES_MEMORY_REQUEST: 2Gi
 265     BUILD_DIR: release-doc-builds
 266   script:
 267     - VALID_BUILD=true
 268     - VERSION=`cat version.json |
 269       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 270     - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
 271       VERSION="$VERSION"-dev ;
 272       fi
 273     - VERSION_FROM_BINARY=`"${BUILD_DIR}"/bin/gmx --version | grep "GROMACS version" | awk '{print $3}'`
 274     - echo "Version name from CMake = $VERSION"
 275     - echo "Version name from binary = $VERSION_FROM_BINARY"
 276     - if [[ "$VERSION" != "$VERSION_FROM_BINARY" && "$GROMACS_RELEASE" == "true" ]] ; then
 277       echo "Version names don't match";
 278       VALID_BUILD=false;
 279       fi
 280     - if [[ "$VALID_BUILD" != "true" ]] ; then
 281       exit 1;
 282       fi
 283   dependencies:
 284     - webpage:dependencies
 285     - prepare-release-version
 286
 287 archive:nightly-webpage:
 288   extends:
 289     - .webpage:build
 290     - .rules:nightly-not-for-release
 291   stage: nightly-deploy
 292   needs:
 293     - job: linkchecker
 294       artifacts: false
 295     - job: webpage:build
 296   variables:
 297     BUILD_DIR: build-docs
 298   before_script:
 299     - eval $(ssh-agent -s)
 300     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
 301     - mkdir -p ~/.ssh
 302     - chmod 700 ~/.ssh
 303     - ssh-keyscan manual.gromacs.org > ~/.ssh/known_hosts # Force overwrite the known hosts so we only have that one key in it.
 304     - chmod 644 ~/.ssh/known_hosts
 305   script:
 306     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 307     - rsync --chmod=u+rwX,g+rwX,o+rX -av $BUILD_DIR/docs/html/* $BUILD_DIR/docs/html/.[a-z]* pbauer@manual.gromacs.org:/var/www/manual/nightly/
 308
 309   artifacts:
 310     when: always
 311     paths:
 312       - webpage.tar.gz
 313
 314 archive:full-release:
 315   extends:
 316     - .webpage:build
 317     - .rules:nightly-only-for-release
 318   stage: release-deploy
 319   dependencies:
 320     - webpage:build:release
 321     - archive:package
 322     - regressiontests:package
 323   variables:
 324     BUILD_DIR: release-doc-builds
 325   script:
 326     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 327   artifacts:
 328     when: always
 329     paths:
 330       - webpage.tar.gz
 331       - gromacs-*tar.gz
 332       - regressiontests-*tar.gz
 333