admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 prepare-release-version:
   4   extends:
   5     - .variables:default
   6     - .rules:nightly-only-for-release
   7   cache: {}
   8   image: gromacs/ci-ubuntu-18.04-llvm-7-docs
   9   stage: configure-build
  10   variables:
  11     KUBERNETES_CPU_LIMIT: 1
  12     KUBERNETES_CPU_REQUEST: 1
  13     KUBERNETES_MEMORY_REQUEST: 2Gi
  14   script:
  15     - cmake -P cmake/gmxVersionInfo.cmake &> version.json
  16   artifacts:
  17     paths:
  18       - version.json
  19
  20 # Special job to package regressiontest files and have them available for testing
  21 # Runs during pre-build
  22 # Set up to only fetch the files and prepare everything for merge requests
  23 regressiontests:prepare:
  24   extends:
  25     - .variables:default
  26     - .rules:merge-and-post-merge-acceptance
  27   cache: {}
  28   image: gromacs/ci-ubuntu-18.04-llvm-7-docs
  29   stage: configure-build
  30   variables:
  31     KUBERNETES_CPU_LIMIT: 1
  32     KUBERNETES_CPU_REQUEST: 1
  33     KUBERNETES_MEMORY_REQUEST: 2Gi
  34   # Always clone the default version for this branch, 2021 in this case
  35   script:
  36     - export REGTESTBRANCH=release-2021
  37     - if [[ ! -z $REGRESSIONTESTBRANCH ]] ; then
  38       export REGTESTBRANCH=$REGRESSIONTESTBRANCH ;
  39       echo "Using $REGTESTBRANCH instead of default" ;
  40       fi
  41     - export CHECKOUTCOMMIT=FETCH_HEAD
  42     - if [[ ! -z $REGRESSIONTESTCOMMIT ]] ; then
  43       export CHECKOUTCOMMIT=$REGRESSIONTESTCOMMIT ;
  44       echo "Using $CHECKOUTCOMMIT instead of head of $REGTESTBRANCH" ;
  45       fi
  46     - if [[ ! -d regressiontests ]] ; then
  47         mkdir regressiontests ;
  48         cd regressiontests ;
  49         git init ;
  50         cd .. ;
  51       fi
  52     - cd regressiontests
  53     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  54     - git checkout -qf $CHECKOUTCOMMIT
  55     - git clean -ffdxq
  56     - git gc
  57     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  58     - echo "Build regressiontests for branch $REGTESTBRANCH"
  59     - mv gmx-regressiontests.tar.gz ..
  60     - cd ..
  61   artifacts:
  62     paths:
  63       - gmx-regressiontests.tar.gz
  64
  65 regressiontests:package:
  66   extends:
  67     - .variables:default
  68     - .rules:nightly-only-for-release
  69   cache: {}
  70   # Docker image uploaded to dockerhub by user eriklindahl
  71   # TODO: Get DockerFile for admin/dockerfiles
  72   image: gromacs/ci-ubuntu-18.04-llvm-7-docs
  73   stage: release-package
  74   variables:
  75     KUBERNETES_CPU_LIMIT: 1
  76     KUBERNETES_CPU_REQUEST: 1
  77     KUBERNETES_MEMORY_REQUEST: 2Gi
  78   needs:
  79     - job: prepare-release-version
  80   script:
  81     - VERSION=`cat version.json |
  82       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
  83     - REGTESTNAME="regressiontests-"$VERSION
  84     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  85       REGTESTNAME=$REGTESTNAME-dev ;
  86       fi
  87     - export REGTESTBRANCH=release-2021
  88     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  89       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  90       fi
  91     - if [[ ! -d regressiontests ]] ; then
  92       mkdir regressiontests ;
  93       cd regressiontests ;
  94       git init ;
  95       cd .. ;
  96       fi
  97     - cd regressiontests
  98     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  99     - git checkout -qf FETCH_HEAD
 100     - git clean -ffdxq
 101     - git gc
 102     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
 103     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
 104     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
 105     - echo "Build regressiontests for branch  $REGTESTBRANCH"
 106     - cd ..
 107   artifacts:
 108     paths:
 109       - regressiontests-*.tar.gz
 110       - gmx-regressiontests.tar.gz
 111
 112 archive:configure:nightly:
 113   extends:
 114     - .docs:configure
 115     - .rules:nightly-not-for-release
 116   stage: nightly-configure
 117   dependencies: []
 118   variables:
 119     BUILD_DIR: build-package
 120     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 121     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 122     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 123
 124 archive:configure:release:
 125   extends:
 126     - .docs:configure
 127     - .rules:nightly-only-for-release
 128   variables:
 129     BUILD_DIR: build-package
 130     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 131     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF -DGMX_USE_SIMD_KERNELS=off
 132     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 133
 134 # Right now we need to use an ugly hack to move the original build directory
 135 # and the ccache files out of the build archive, as it is not possible to use
 136 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 137 # As soon as this is resolved we should use the separate build directories instead of
 138 # manually hacking things. TODO: #3361
 139 .archive:build:
 140   extends:
 141     - .variables:default
 142     - .gromacs:base:build
 143     - .use-ccache
 144     - .before_script:default
 145     - .docs:build
 146   image: gromacs/ci-ubuntu-18.04-llvm-7-docs
 147   variables:
 148     KUBERNETES_CPU_LIMIT: 4
 149     KUBERNETES_CPU_REQUEST: 2
 150     KUBERNETES_MEMORY_REQUEST: 4Gi
 151   script:
 152     - rm -rf gromacs*tar.gz
 153     - cd $BUILD_DIR
 154     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 155     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 156     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 157     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 158     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 159     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 160     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 161       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 162       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 163     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 164       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 165       packageBuildLogFile.log | tee -a buildErrors.log || true
 166     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 167     - mv gromacs*tar.gz ..
 168     - cd ..
 169     # ugly hack begins here
 170     - mkdir tempdir
 171     - SOURCETARBALL=`ls gromacs*tar.gz`
 172     - mv $SOURCETARBALL tempdir
 173     - cd tempdir
 174     - tar -xf $SOURCETARBALL
 175     - rm -rf gromacs-*/$BUILD_DIR
 176     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 177     - mv $SOURCETARBALL ..
 178     - cd ..
 179     - rm -rf tempdir
 180     # Hack ends here
 181   artifacts:
 182     when: always
 183     expire_in: 1 week
 184     paths:
 185       - $BUILD_DIR/*log
 186       - gromacs*tar.gz
 187
 188 archive:build:
 189   extends:
 190     - .archive:build
 191     - .rules:nightly-not-for-release
 192   stage: nightly-build
 193   needs:
 194     - job: archive:configure:nightly
 195   variables:
 196     BUILD_DIR: build-package
 197
 198 archive:package:
 199   extends:
 200     - .archive:build
 201     - .rules:nightly-only-for-release
 202   stage: release-package
 203   needs:
 204     - job: archive:configure:release
 205   variables:
 206     BUILD_DIR: build-package
 207
 208 release-verify:
 209   image: gromacs/ci-ubuntu-18.04-llvm-7-docs
 210   stage: release-verify
 211   extends:
 212     - .variables:default
 213     - .rules:nightly-only-for-release
 214   cache: {}
 215
 216   variables:
 217     KUBERNETES_CPU_LIMIT: 1
 218     KUBERNETES_CPU_REQUEST: 1
 219     KUBERNETES_MEMORY_REQUEST: 2Gi
 220   script:
 221     - VERSION=`cat version.json |
 222       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 223     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 224       VERSION=$VERSION-dev ;
 225       fi
 226     - REGTEST_COMPARE=`cat version.json |
 227       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 228     - SOURCENAME=gromacs-$VERSION
 229     - SOURCETARBALL=$SOURCENAME.tar.gz
 230     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 231     - REGTESTNAME=regressiontests-$VERSION
 232     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 233     - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'`
 234     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 235     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 236     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 237     - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then
 238       echo "Mismatch in regressiontest md5sums";
 239       exit 1;
 240       fi
 241   dependencies:
 242     - archive:package
 243     - regressiontests:package
 244     - prepare-release-version
 245
 246 archive:nightly-webpage:
 247   extends:
 248     - .webpage:build
 249     - .rules:nightly-not-for-release
 250   stage: nightly-deploy
 251   needs:
 252     - job: linkchecker
 253       artifacts: false
 254     - job: webpage:build
 255   variables:
 256     BUILD_DIR: build-docs
 257   script:
 258     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 259
 260   artifacts:
 261     when: always
 262     paths:
 263       - webpage.tar.gz
 264
 265 archive:full-release:
 266   extends:
 267     - .webpage:build
 268     - .rules:nightly-only-for-release
 269   stage: release-deploy
 270   dependencies:
 271     - webpage:build:release
 272     - archive:package
 273     - regressiontests:package
 274   variables:
 275     BUILD_DIR: release-doc-builds
 276   script:
 277     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 278   artifacts:
 279     when: always
 280     paths:
 281       - webpage.tar.gz
 282       - gromacs-*tar.gz
 283       - regressiontests-*tar.gz
 284