admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 # Special job to run cmake to output the version string and
   4 # regressiontest md5sum to a file
   5
   6 prepare-release-version:
   7   extends:
   8     - .variables:default
   9     - .rules:nightly-only-for-release
  10   cache: {}
  11   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  12   stage: configure-build
  13   variables:
  14     KUBERNETES_CPU_LIMIT: 1
  15     KUBERNETES_CPU_REQUEST: 1
  16     KUBERNETES_MEMORY_REQUEST: 2Gi
  17   script:
  18     - cmake . -DGMX_BUILD_TARBALL=on
  19   artifacts:
  20     paths:
  21       - VersionInfo-partial.cmake.cmakein
  22
  23 # Special job to package regressiontest files and have them available for testing
  24 # Runs during pre-build
  25 # Set up to only fetch the files and prepare everything for merge requests
  26 regressiontests:prepare:
  27   extends:
  28     - .variables:default
  29     - .rules:merge-and-post-merge-acceptance
  30   cache: {}
  31   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  32   stage: configure-build
  33   variables:
  34     KUBERNETES_CPU_LIMIT: 1
  35     KUBERNETES_CPU_REQUEST: 1
  36     KUBERNETES_MEMORY_REQUEST: 2Gi
  37   # Always clone the default version for this branch, master in this case
  38   script:
  39     - export REGTESTBRANCH=master
  40     - if [[ ! -z $REGRESSIONTESTBRANCH ]] ; then
  41       export REGTESTBRANCH=$REGRESSIONTESTBRANCH ;
  42       echo "Using $REGTESTBRANCH instead of default" ;
  43       fi
  44     - export CHECKOUTCOMMIT=FETCH_HEAD
  45     - if [[ ! -z $REGRESSIONTESTCOMMIT ]] ; then
  46       export CHECKOUTCOMMIT=$REGRESSIONTESTCOMMIT ;
  47       echo "Using $CHECKOUTCOMMIT instead of head of $REGTESTBRANCH" ;
  48       fi
  49     - if [[ ! -d regressiontests ]] ; then
  50         mkdir regressiontests ;
  51         cd regressiontests ;
  52         git init ;
  53         cd .. ;
  54       fi
  55     - cd regressiontests
  56     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  57     - git checkout -qf $CHECKOUTCOMMIT
  58     - git clean -ffdxq
  59     - git gc
  60     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  61     - echo "Build regressiontests for branch $REGTESTBRANCH"
  62     - mv gmx-regressiontests.tar.gz ..
  63     - cd ..
  64   artifacts:
  65     paths:
  66       - gmx-regressiontests.tar.gz
  67
  68 regressiontests:package:
  69   extends:
  70     - .variables:default
  71     - .rules:nightly-only-for-release
  72   cache: {}
  73   # Docker image uploaded to dockerhub by user eriklindahl
  74   # TODO: Get DockerFile for admin/dockerfiles
  75   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  76   stage: release-package
  77   variables:
  78     KUBERNETES_CPU_LIMIT: 1
  79     KUBERNETES_CPU_REQUEST: 1
  80     KUBERNETES_MEMORY_REQUEST: 2Gi
  81   needs:
  82     - job: prepare-release-version
  83   script:
  84     - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
  85     - REGTESTNAME="regressiontests-"$VERSION
  86     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  87       REGTESTNAME=$REGTESTNAME-dev ;
  88       fi
  89     - export REGTESTBRANCH=master
  90     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  91       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  92       fi
  93     - if [[ ! -d regressiontests ]] ; then
  94       mkdir regressiontests ;
  95       cd regressiontests ;
  96       git init ;
  97       cd .. ;
  98       fi
  99     - cd regressiontests
 100     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
 101     - git checkout -qf FETCH_HEAD
 102     - git clean -ffdxq
 103     - git gc
 104     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
 105     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
 106     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
 107     - echo "Build regressiontests $REGTESTNAME for branch  $REGTESTBRANCH"
 108     - cd ..
 109   artifacts:
 110     paths:
 111       - regressiontests-*.tar.gz
 112       - gmx-regressiontests.tar.gz
 113
 114 archive:configure:nightly:
 115   extends:
 116     - .docs:configure
 117     - .rules:nightly-not-for-release
 118   stage: nightly-configure
 119   dependencies: []
 120   variables:
 121     BUILD_DIR: build-package
 122     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 123     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 124     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 125
 126 archive:configure:release:
 127   extends:
 128     - .docs:configure
 129     - .rules:nightly-only-for-release
 130   variables:
 131     BUILD_DIR: build-package
 132     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 133     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 134     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 135
 136 # Right now we need to use an ugly hack to move the original build directory
 137 # and the ccache files out of the build archive, as it is not possible to use
 138 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 139 # As soon as this is resolved we should use the separate build directories instead of
 140 # manually hacking things. TODO: #3361
 141 .archive:build:
 142   extends:
 143     - .variables:default
 144     - .gromacs:base:build
 145     - .use-ccache
 146     - .before_script:default
 147     - .docs:build
 148   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 149   variables:
 150     KUBERNETES_CPU_LIMIT: 4
 151     KUBERNETES_CPU_REQUEST: 2
 152     KUBERNETES_MEMORY_REQUEST: 4Gi
 153   script:
 154     - rm -rf gromacs*tar.gz
 155     - cd $BUILD_DIR
 156     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 157     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 158     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 159     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 160     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 161     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 162     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 163       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 164       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 165     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 166       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 167       packageBuildLogFile.log | tee -a buildErrors.log || true
 168     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 169     - mv gromacs*tar.gz ..
 170     - cd ..
 171     # ugly hack begins here
 172     - mkdir tempdir
 173     - SOURCETARBALL=`ls gromacs*tar.gz`
 174     - mv $SOURCETARBALL tempdir
 175     - cd tempdir
 176     - tar -xf $SOURCETARBALL
 177     - rm -rf gromacs-*/$BUILD_DIR
 178     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 179     - mv $SOURCETARBALL ..
 180     - cd ..
 181     - rm -rf tempdir
 182     # Hack ends here
 183   artifacts:
 184     when: always
 185     expire_in: 1 week
 186     paths:
 187       - $BUILD_DIR/*logs
 188       - $BUILD_DIR/bin
 189       - $BUILD_DIR/lib
 190       - gromacs*tar.gz
 191
 192 archive:build:
 193   extends:
 194     - .archive:build
 195     - .rules:nightly-not-for-release
 196   stage: nightly-build
 197   needs:
 198     - job: archive:configure:nightly
 199   variables:
 200     BUILD_DIR: build-package
 201
 202 archive:package:
 203   extends:
 204     - .archive:build
 205     - .rules:nightly-only-for-release
 206   stage: release-package
 207   needs:
 208     - job: archive:configure:release
 209   variables:
 210     BUILD_DIR: build-package
 211
 212 checksum-verify:
 213   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 214   stage: release-verify
 215   extends:
 216     - .variables:default
 217     - .rules:nightly-only-for-release
 218   cache: {}
 219
 220   variables:
 221     KUBERNETES_CPU_LIMIT: 1
 222     KUBERNETES_CPU_REQUEST: 1
 223     KUBERNETES_MEMORY_REQUEST: 2Gi
 224     BUILD_DIR: build-package
 225   script:
 226     - VALID_BUILD=true
 227     - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
 228     - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
 229       VERSION="$VERSION"-dev ;
 230       fi
 231     - REGTEST_COMPARE=`sed -n -e '/REGRESSIONTEST_MD5SUM/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
 232     - SOURCENAME=gromacs-"$VERSION"
 233     - SOURCETARBALL="$SOURCENAME".tar.gz
 234     - SOURCE_MD5SUM=`md5sum "$SOURCETARBALL" | awk '{print $1}'`
 235     - REGTESTNAME=regressiontests-"$VERSION"
 236     - REGTESTTARBALL="$REGTESTNAME".tar.gz
 237     - REGTEST_MD5SUM=`md5sum "$REGTESTTARBALL" | awk '{print $1}'`
 238     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 239     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 240     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 241     - if [[ "$REGTEST_COMPARE" != "$REGTEST_MD5SUM" && "$GROMACS_RELEASE" == "true" ]] ; then
 242       echo "Mismatch in regressiontest md5sums";
 243       VALID_BUILD=false;
 244       fi
 245     - if [[ "$VALID_BUILD" != "true" ]] ; then
 246       exit 1;
 247       fi
 248   dependencies:
 249     - archive:package
 250     - regressiontests:package
 251     - prepare-release-version
 252
 253 version-verify:
 254   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 255   stage: release-verify
 256   extends:
 257     - .variables:default
 258     - .rules:nightly-only-for-release
 259   cache: {}
 260
 261   variables:
 262     KUBERNETES_CPU_LIMIT: 1
 263     KUBERNETES_CPU_REQUEST: 1
 264     KUBERNETES_MEMORY_REQUEST: 2Gi
 265     BUILD_DIR: release-doc-builds
 266   script:
 267     - VALID_BUILD=true
 268     - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
 269     - if [[ "$GROMACS_RELEASE" != "true" ]] ; then
 270       VERSION="$VERSION"-dev ;
 271       fi
 272     - VERSION_FROM_BINARY=`"${BUILD_DIR}"/bin/gmx --version | grep "GROMACS version" | awk '{print $3}'`
 273     - echo "Version name from CMake = $VERSION"
 274     - echo "Version name from binary = $VERSION_FROM_BINARY"
 275     - if [[ "$VERSION" != "$VERSION_FROM_BINARY" && "$GROMACS_RELEASE" == "true" ]] ; then
 276       echo "Version names don't match";
 277       VALID_BUILD=false;
 278       fi
 279     - if [[ "$VALID_BUILD" != "true" ]] ; then
 280       exit 1;
 281       fi
 282   dependencies:
 283     - webpage:dependencies
 284     - prepare-release-version
 285
 286 archive:nightly-webpage:
 287   extends:
 288     - .webpage:build
 289     - .rules:nightly-not-for-release
 290   stage: nightly-deploy
 291   needs:
 292     - job: linkchecker
 293       artifacts: false
 294     - job: webpage:build
 295   variables:
 296     BUILD_DIR: build-docs
 297   before_script:
 298     - eval $(ssh-agent -s)
 299     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
 300     - mkdir -p ~/.ssh
 301     - chmod 700 ~/.ssh
 302     - ssh-keyscan manual.gromacs.org > ~/.ssh/known_hosts # Force overwrite the known hosts so we only have that one key in it.
 303     - chmod 644 ~/.ssh/known_hosts
 304   script:
 305     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 306     - rsync --chmod=u+rwX,g+rwX,o+rX -av $BUILD_DIR/docs/html/* $BUILD_DIR/docs/html/.[a-z]* pbauer@manual.gromacs.org:/var/www/manual/nightly/
 307
 308   artifacts:
 309     when: always
 310     paths:
 311       - webpage.tar.gz
 312
 313 archive:full-release:
 314   extends:
 315     - .webpage:build
 316     - .rules:nightly-only-for-release
 317   stage: release-deploy
 318   dependencies:
 319     - webpage:build:release
 320     - archive:package
 321     - regressiontests:package
 322   variables:
 323     BUILD_DIR: release-doc-builds
 324   script:
 325     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 326   artifacts:
 327     when: always
 328     paths:
 329       - webpage.tar.gz
 330       - gromacs-*tar.gz
 331       - regressiontests-*tar.gz
 332