admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 prepare-release-version:
   4   extends:
   5     - .variables:default
   6     - .rules:nightly-only-for-release
   7   cache: {}
   8   # Docker image uploaded to dockerhub by user eriklindahl
   9   # TODO: Get DockerFile for admin/dockerfiles
  10   image: biophysics/gcc-gromacs
  11   stage: configure-build
  12
  13   variables:
  14     KUBERNETES_CPU_LIMIT: 1
  15     KUBERNETES_CPU_REQUEST: 1
  16     KUBERNETES_MEMORY_LIMIT: 2Gi
  17   script:
  18     - cmake -P cmake/gmxVersionInfo.cmake &> version.json
  19
  20   artifacts:
  21     paths:
  22       - version.json
  23
  24 # Special job to package regressiontest files and have them available for testing
  25 # Runs during pre-build
  26 # Set up to only fetch the files and prepare everything for merge requests
  27 regressiontests:prepare:
  28   extends:
  29     - .variables:default
  30     - .rules:merge-requests
  31   cache: {}
  32   image: biophysics/gcc-gromacs
  33   stage: configure-build
  34
  35   variables:
  36     KUBERNETES_CPU_LIMIT: 1
  37     KUBERNETES_CPU_REQUEST: 1
  38     KUBERNETES_MEMORY_LIMIT: 2Gi
  39   # Always clone the default version for this branch, master in this case
  40   script:
  41     - export REGTESTBRANCH=master
  42     - if [[ ! -d regressiontests ]] ; then
  43         mkdir regressiontests ;
  44         cd regressiontests ;
  45         git init ;
  46         cd .. ;
  47       fi
  48     - cd regressiontests
  49     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  50     - git checkout -qf FETCH_HEAD
  51     - git clean -ffdxq
  52     - git gc
  53     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  54     - echo "Build regressiontests for branch $REGTESTBRANCH"
  55     - mv gmx-regressiontests.tar.gz ..
  56     - cd ..
  57   artifacts:
  58     paths:
  59       - gmx-regressiontests.tar.gz
  60
  61 regressiontests:package:
  62   extends:
  63     - .variables:default
  64     - .rules:nightly-only-for-release
  65   cache: {}
  66   # Docker image uploaded to dockerhub by user eriklindahl
  67   # TODO: Get DockerFile for admin/dockerfiles
  68   image: biophysics/gcc-gromacs
  69   stage: release-package
  70
  71   variables:
  72     KUBERNETES_CPU_LIMIT: 1
  73     KUBERNETES_CPU_REQUEST: 1
  74     KUBERNETES_MEMORY_LIMIT: 2Gi
  75   needs:
  76     - job: prepare-release-version
  77   script:
  78     - VERSION=`cat version.json |
  79       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
  80     - REGTESTNAME="regressiontests-"$VERSION
  81     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  82       REGTESTNAME=$REGTESTNAME-dev ;
  83       fi
  84     - export REGTESTBRANCH=master
  85     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  86       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  87       fi
  88     - if [[ ! -d regressiontests ]] ; then
  89       mkdir regressiontests ;
  90       cd regressiontests ;
  91       git init ;
  92       cd .. ;
  93       fi
  94     - cd regressiontests
  95     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  96     - git checkout -qf FETCH_HEAD
  97     - git clean -ffdxq
  98     - git gc
  99     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
 100     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
 101     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
 102     - echo "Build regressiontests for branch  $REGTESTBRANCH"
 103     - cd ..
 104   artifacts:
 105     paths:
 106       - regressiontests-*.tar.gz
 107       - gmx-regressiontests.tar.gz
 108
 109 archive:configure:nightly:
 110   extends:
 111     - .docs:configure
 112     - .rules:nightly-not-for-release
 113   stage: nightly-configure
 114   dependencies: []
 115   variables:
 116     BUILD_DIR: build-package
 117     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 118     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
 119     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 120
 121 archive:configure:release:
 122   extends:
 123     - .docs:configure
 124     - .rules:nightly-only-for-release
 125   variables:
 126     BUILD_DIR: build-package
 127     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 128     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
 129     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 130
 131 # Right now we need to use an ugly hack to move the original build directory
 132 # and the ccache files out of the build archive, as it is not possible to use
 133 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 134 # As soon as this is resolved we should use the separate build directories instead of
 135 # manually hacking things. TODO: #3361
 136 .archive:build:
 137   extends:
 138     - .variables:default
 139     - .gromacs:base:build
 140     - .use-ccache
 141     - .before_script:default
 142     - .docs:build
 143   image: gromacs/ci-docs-llvm:master
 144   variables:
 145     KUBERNETES_CPU_LIMIT: 4
 146     KUBERNETES_CPU_REQUEST: 2
 147     KUBERNETES_MEMORY_LIMIT: 4Gi
 148   script:
 149     - rm -rf gromacs*tar.gz
 150     - cd $BUILD_DIR
 151     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 152     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 153     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 154     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 155     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 156     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 157     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 158       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 159       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 160     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 161       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 162       packageBuildLogFile.log | tee -a buildErrors.log || true
 163     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 164     - mv gromacs*tar.gz ..
 165     - cd ..
 166     # ugly hack begins here
 167     - mkdir tempdir
 168     - SOURCETARBALL=`ls gromacs*tar.gz`
 169     - mv $SOURCETARBALL tempdir
 170     - cd tempdir
 171     - tar -xf $SOURCETARBALL
 172     - rm -rf gromacs-*/$BUILD_DIR
 173     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 174     - mv $SOURCETARBALL ..
 175     - cd ..
 176     - rm -rf tempdir
 177     # Hack ends here
 178   artifacts:
 179     when: always
 180     expire_in: 1 week
 181     paths:
 182       - $BUILD_DIR/*log
 183       - gromacs*tar.gz
 184
 185 archive:build:
 186   extends:
 187     - .archive:build
 188     - .rules:nightly-not-for-release
 189   stage: nightly-build
 190   needs:
 191     - job: archive:configure:nightly
 192   variables:
 193     BUILD_DIR: build-package
 194
 195 archive:package:
 196   extends:
 197     - .archive:build
 198     - .rules:nightly-only-for-release
 199   stage: release-package
 200   needs:
 201     - job: archive:configure:release
 202   variables:
 203     BUILD_DIR: build-package
 204
 205 release-verify:
 206   image: biophysics/gcc-gromacs
 207   stage: release-verify
 208   extends:
 209     - .variables:default
 210     - .rules:nightly-only-for-release
 211   cache: {}
 212
 213   variables:
 214     KUBERNETES_CPU_LIMIT: 1
 215     KUBERNETES_CPU_REQUEST: 1
 216     KUBERNETES_MEMORY_LIMIT: 2Gi
 217   script:
 218     - VERSION=`cat version.json |
 219       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 220     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 221       VERSION=$VERSION-dev ;
 222       fi
 223     - REGTEST_COMPARE=`cat version.json |
 224       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 225     - SOURCENAME=gromacs-$VERSION
 226     - SOURCETARBALL=$SOURCENAME.tar.gz
 227     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 228     - REGTESTNAME=regressiontests-$VERSION
 229     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 230     - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'`
 231     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 232     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 233     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 234     - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then
 235       echo "Mismatch in regressiontest md5sums";
 236       exit 1;
 237       fi
 238   dependencies:
 239     - archive:package
 240     - regressiontests:package
 241     - prepare-release-version
 242
 243 archive:nightly-webpage:
 244   extends:
 245     - .webpage:build
 246     - .rules:nightly-not-for-release
 247   stage: nightly-deploy
 248   needs:
 249     - job: linkchecker
 250       artifacts: false
 251     - job: webpage:build
 252   variables:
 253     BUILD_DIR: build-docs
 254   script:
 255     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 256   artifacts:
 257     when: always
 258     paths:
 259       - webpage.tar.gz
 260
 261 archive:full-release:
 262   extends:
 263     - .webpage:build
 264     - .rules:nightly-only-for-release
 265   stage: release-deploy
 266   dependencies:
 267     - webpage:build:release
 268     - archive:package
 269     - regressiontests:package
 270   variables:
 271     BUILD_DIR: release-doc-builds
 272   script:
 273     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 274   artifacts:
 275     when: always
 276     paths:
 277       - webpage.tar.gz
 278       - gromacs-*tar.gz
 279       - regressiontests-*tar.gz
 280