admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 prepare-release-version:
   4   extends:
   5     - .variables:default
   6     - .rules:nightly-only-for-release
   7   cache: {}
   8   # Docker image uploaded to dockerhub by user eriklindahl
   9   # TODO: Get DockerFile for admin/dockerfiles
  10   image: gromacs/ci-docs-llvm-master
  11   stage: configure-build
  12   variables:
  13     KUBERNETES_CPU_LIMIT: 1
  14     KUBERNETES_CPU_REQUEST: 1
  15     KUBERNETES_MEMORY_REQUEST: 2Gi
  16   script:
  17     - cmake -P cmake/gmxVersionInfo.cmake &> version.json
  18   artifacts:
  19     paths:
  20       - version.json
  21
  22 # Special job to package regressiontest files and have them available for testing
  23 # Runs during pre-build
  24 # Set up to only fetch the files and prepare everything for merge requests
  25 regressiontests:prepare:
  26   extends:
  27     - .variables:default
  28     - .rules:merge-requests
  29   cache: {}
  30   image: gromacs/ci-docs-llvm-master
  31   stage: configure-build
  32   variables:
  33     KUBERNETES_CPU_LIMIT: 1
  34     KUBERNETES_CPU_REQUEST: 1
  35     KUBERNETES_MEMORY_REQUEST: 2Gi
  36   # Always clone the default version for this branch, master in this case
  37   script:
  38     - export REGTESTBRANCH=master
  39     - if [[ -z $REGRESSIONTESTBRANCH ]] ; then
  40       export REGTESTBRANCH=$REGRESSIONTESTBRANCH ;
  41       echo "Using $REGTESTBRANCH instead of default" ;
  42       fi
  43     - export CHECKOUTCOMMIT=FETCH_HEAD
  44     - if [[ ! -z $REGRESSIONTESTCOMMIT ]] ; then
  45       export CHECKOUTCOMMIT=$REGRESSIONTESTCOMMIT ;
  46       echo "Using $CHECKOUTCOMMIT instead of head of $REGTESTBRANCH" ;
  47       fi
  48     - if [[ ! -d regressiontests ]] ; then
  49         mkdir regressiontests ;
  50         cd regressiontests ;
  51         git init ;
  52         cd .. ;
  53       fi
  54     - cd regressiontests
  55     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  56     - git checkout -qf $CHECKOUTCOMMIT
  57     - git clean -ffdxq
  58     - git gc
  59     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  60     - echo "Build regressiontests for branch $REGTESTBRANCH"
  61     - mv gmx-regressiontests.tar.gz ..
  62     - cd ..
  63   artifacts:
  64     paths:
  65       - gmx-regressiontests.tar.gz
  66
  67 regressiontests:package:
  68   extends:
  69     - .variables:default
  70     - .rules:nightly-only-for-release
  71   cache: {}
  72   # Docker image uploaded to dockerhub by user eriklindahl
  73   # TODO: Get DockerFile for admin/dockerfiles
  74   image: gromacs/ci-docs-llvm-master
  75   stage: release-package
  76   variables:
  77     KUBERNETES_CPU_LIMIT: 1
  78     KUBERNETES_CPU_REQUEST: 1
  79     KUBERNETES_MEMORY_REQUEST: 2Gi
  80   needs:
  81     - job: prepare-release-version
  82   script:
  83     - VERSION=`cat version.json |
  84       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
  85     - REGTESTNAME="regressiontests-"$VERSION
  86     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  87       REGTESTNAME=$REGTESTNAME-dev ;
  88       fi
  89     - export REGTESTBRANCH=master
  90     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  91       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  92       fi
  93     - if [[ ! -d regressiontests ]] ; then
  94       mkdir regressiontests ;
  95       cd regressiontests ;
  96       git init ;
  97       cd .. ;
  98       fi
  99     - cd regressiontests
 100     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
 101     - git checkout -qf FETCH_HEAD
 102     - git clean -ffdxq
 103     - git gc
 104     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
 105     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
 106     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
 107     - echo "Build regressiontests for branch  $REGTESTBRANCH"
 108     - cd ..
 109   artifacts:
 110     paths:
 111       - regressiontests-*.tar.gz
 112       - gmx-regressiontests.tar.gz
 113
 114 archive:configure:nightly:
 115   extends:
 116     - .docs:configure
 117     - .rules:nightly-not-for-release
 118   stage: nightly-configure
 119   dependencies: []
 120   variables:
 121     BUILD_DIR: build-package
 122     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 123     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 124     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 125
 126 archive:configure:release:
 127   extends:
 128     - .docs:configure
 129     - .rules:nightly-only-for-release
 130   variables:
 131     BUILD_DIR: build-package
 132     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 133     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 134     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 135
 136 # Right now we need to use an ugly hack to move the original build directory
 137 # and the ccache files out of the build archive, as it is not possible to use
 138 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 139 # As soon as this is resolved we should use the separate build directories instead of
 140 # manually hacking things. TODO: #3361
 141 .archive:build:
 142   extends:
 143     - .variables:default
 144     - .gromacs:base:build
 145     - .use-ccache
 146     - .before_script:default
 147     - .docs:build
 148   image: gromacs/ci-docs-llvm-master
 149   variables:
 150     KUBERNETES_CPU_LIMIT: 4
 151     KUBERNETES_CPU_REQUEST: 2
 152     KUBERNETES_MEMORY_REQUEST: 4Gi
 153   script:
 154     - rm -rf gromacs*tar.gz
 155     - cd $BUILD_DIR
 156     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 157     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 158     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 159     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 160     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 161     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 162     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 163       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 164       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 165     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 166       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 167       packageBuildLogFile.log | tee -a buildErrors.log || true
 168     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 169     - mv gromacs*tar.gz ..
 170     - cd ..
 171     # ugly hack begins here
 172     - mkdir tempdir
 173     - SOURCETARBALL=`ls gromacs*tar.gz`
 174     - mv $SOURCETARBALL tempdir
 175     - cd tempdir
 176     - tar -xf $SOURCETARBALL
 177     - rm -rf gromacs-*/$BUILD_DIR
 178     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 179     - mv $SOURCETARBALL ..
 180     - cd ..
 181     - rm -rf tempdir
 182     # Hack ends here
 183   artifacts:
 184     when: always
 185     expire_in: 1 week
 186     paths:
 187       - $BUILD_DIR/*log
 188       - gromacs*tar.gz
 189
 190 archive:build:
 191   extends:
 192     - .archive:build
 193     - .rules:nightly-not-for-release
 194   stage: nightly-build
 195   needs:
 196     - job: archive:configure:nightly
 197   variables:
 198     BUILD_DIR: build-package
 199
 200 archive:package:
 201   extends:
 202     - .archive:build
 203     - .rules:nightly-only-for-release
 204   stage: release-package
 205   needs:
 206     - job: archive:configure:release
 207   variables:
 208     BUILD_DIR: build-package
 209
 210 release-verify:
 211   image: gromacs/ci-docs-llvm-master
 212   stage: release-verify
 213   extends:
 214     - .variables:default
 215     - .rules:nightly-only-for-release
 216   cache: {}
 217
 218   variables:
 219     KUBERNETES_CPU_LIMIT: 1
 220     KUBERNETES_CPU_REQUEST: 1
 221     KUBERNETES_MEMORY_REQUEST: 2Gi
 222   script:
 223     - VERSION=`cat version.json |
 224       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 225     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 226       VERSION=$VERSION-dev ;
 227       fi
 228     - REGTEST_COMPARE=`cat version.json |
 229       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 230     - SOURCENAME=gromacs-$VERSION
 231     - SOURCETARBALL=$SOURCENAME.tar.gz
 232     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 233     - REGTESTNAME=regressiontests-$VERSION
 234     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 235     - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'`
 236     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 237     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 238     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 239     - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then
 240       echo "Mismatch in regressiontest md5sums";
 241       exit 1;
 242       fi
 243   dependencies:
 244     - archive:package
 245     - regressiontests:package
 246     - prepare-release-version
 247
 248 archive:nightly-webpage:
 249   extends:
 250     - .webpage:build
 251     - .rules:nightly-not-for-release
 252   stage: nightly-deploy
 253   needs:
 254     - job: linkchecker
 255       artifacts: false
 256     - job: webpage:build
 257   variables:
 258     BUILD_DIR: build-docs
 259   before_script:
 260     - eval $(ssh-agent -s)
 261     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
 262     - mkdir -p ~/.ssh
 263     - chmod 700 ~/.ssh
 264     - ssh-keyscan www.gromacs.org > ~/.ssh/known_hosts # Force overwrite the known hosts so we only have that one key in it.
 265     - chmod 644 ~/.ssh/known_hosts
 266   script:
 267     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 268     - rsync --chmod=u+rwX,g+rwX,o+rX -av $BUILD_DIR/docs/html/* $BUILD_DIR/docs/html/.[a-z]* pbauer@www.gromacs.org:/var/www/manual/nightly/
 269
 270   artifacts:
 271     when: always
 272     paths:
 273       - webpage.tar.gz
 274
 275 archive:full-release:
 276   extends:
 277     - .webpage:build
 278     - .rules:nightly-only-for-release
 279   stage: release-deploy
 280   dependencies:
 281     - webpage:build:release
 282     - archive:package
 283     - regressiontests:package
 284   variables:
 285     BUILD_DIR: release-doc-builds
 286   script:
 287     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 288   artifacts:
 289     when: always
 290     paths:
 291       - webpage.tar.gz
 292       - gromacs-*tar.gz
 293       - regressiontests-*tar.gz
 294