admin/gitlab-ci/archive.gitlab-ci.yml

   1 # Packages, exported artifacts, and release engineering processes.
   2
   3 prepare-release-version:
   4   extends:
   5     - .variables:default
   6     - .rules:nightly-only-for-release
   7   cache: {}
   8   # Docker image uploaded to dockerhub by user eriklindahl
   9   # TODO: Get DockerFile for admin/dockerfiles
  10   image: biophysics/gcc-gromacs
  11   stage: configure-build
  12   variables:
  13     KUBERNETES_CPU_LIMIT: 1
  14     KUBERNETES_CPU_REQUEST: 1
  15     KUBERNETES_MEMORY_REQUEST: 2Gi
  16   script:
  17     - cmake -P cmake/gmxVersionInfo.cmake &> version.json
  18   artifacts:
  19     paths:
  20       - version.json
  21
  22 # Special job to package regressiontest files and have them available for testing
  23 # Runs during pre-build
  24 # Set up to only fetch the files and prepare everything for merge requests
  25 regressiontests:prepare:
  26   extends:
  27     - .variables:default
  28     - .rules:merge-requests
  29   cache: {}
  30   image: biophysics/gcc-gromacs
  31   stage: configure-build
  32   variables:
  33     KUBERNETES_CPU_LIMIT: 1
  34     KUBERNETES_CPU_REQUEST: 1
  35     KUBERNETES_MEMORY_REQUEST: 2Gi
  36   # Always clone the default version for this branch, master in this case
  37   script:
  38     - export REGTESTBRANCH=master
  39     - if [[ ! -d regressiontests ]] ; then
  40         mkdir regressiontests ;
  41         cd regressiontests ;
  42         git init ;
  43         cd .. ;
  44       fi
  45     - cd regressiontests
  46     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  47     - git checkout -qf FETCH_HEAD
  48     - git clean -ffdxq
  49     - git gc
  50     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  51     - echo "Build regressiontests for branch $REGTESTBRANCH"
  52     - mv gmx-regressiontests.tar.gz ..
  53     - cd ..
  54   artifacts:
  55     paths:
  56       - gmx-regressiontests.tar.gz
  57
  58 regressiontests:package:
  59   extends:
  60     - .variables:default
  61     - .rules:nightly-only-for-release
  62   cache: {}
  63   # Docker image uploaded to dockerhub by user eriklindahl
  64   # TODO: Get DockerFile for admin/dockerfiles
  65   image: biophysics/gcc-gromacs
  66   stage: release-package
  67   variables:
  68     KUBERNETES_CPU_LIMIT: 1
  69     KUBERNETES_CPU_REQUEST: 1
  70     KUBERNETES_MEMORY_REQUEST: 2Gi
  71   needs:
  72     - job: prepare-release-version
  73   script:
  74     - VERSION=`cat version.json |
  75       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
  76     - REGTESTNAME="regressiontests-"$VERSION
  77     - if [[ $GROMACS_RELEASE != "true" ]] ; then
  78       REGTESTNAME=$REGTESTNAME-dev ;
  79       fi
  80     - export REGTESTBRANCH=master
  81     - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then
  82       export REGTESTBRANCH=$CI_COMMIT_REF_NAME ;
  83       fi
  84     - if [[ ! -d regressiontests ]] ; then
  85       mkdir regressiontests ;
  86       cd regressiontests ;
  87       git init ;
  88       cd .. ;
  89       fi
  90     - cd regressiontests
  91     - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH
  92     - git checkout -qf FETCH_HEAD
  93     - git clean -ffdxq
  94     - git gc
  95     - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD
  96     - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD
  97     - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz ..
  98     - echo "Build regressiontests for branch  $REGTESTBRANCH"
  99     - cd ..
 100   artifacts:
 101     paths:
 102       - regressiontests-*.tar.gz
 103       - gmx-regressiontests.tar.gz
 104
 105 archive:configure:nightly:
 106   extends:
 107     - .docs:configure
 108     - .rules:nightly-not-for-release
 109   stage: nightly-configure
 110   dependencies: []
 111   variables:
 112     BUILD_DIR: build-package
 113     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 114     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
 115     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 116
 117 archive:configure:release:
 118   extends:
 119     - .docs:configure
 120     - .rules:nightly-only-for-release
 121   variables:
 122     BUILD_DIR: build-package
 123     CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None
 124     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF
 125     CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF
 126
 127 # Right now we need to use an ugly hack to move the original build directory
 128 # and the ccache files out of the build archive, as it is not possible to use
 129 # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203
 130 # As soon as this is resolved we should use the separate build directories instead of
 131 # manually hacking things. TODO: #3361
 132 .archive:build:
 133   extends:
 134     - .variables:default
 135     - .gromacs:base:build
 136     - .use-ccache
 137     - .before_script:default
 138     - .docs:build
 139   image: gromacs/ci-docs-llvm:master
 140   variables:
 141     KUBERNETES_CPU_LIMIT: 4
 142     KUBERNETES_CPU_REQUEST: 2
 143     KUBERNETES_MEMORY_REQUEST: 4Gi
 144   script:
 145     - rm -rf gromacs*tar.gz
 146     - cd $BUILD_DIR
 147     - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log
 148     - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log
 149     - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log
 150     - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log
 151     - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log
 152     - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log
 153     - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log
 154       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 155       packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true
 156     - grep "cannot be built" buildLogFile.log manBuildLogFile.log
 157       completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log
 158       packageBuildLogFile.log | tee -a buildErrors.log || true
 159     - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi
 160     - mv gromacs*tar.gz ..
 161     - cd ..
 162     # ugly hack begins here
 163     - mkdir tempdir
 164     - SOURCETARBALL=`ls gromacs*tar.gz`
 165     - mv $SOURCETARBALL tempdir
 166     - cd tempdir
 167     - tar -xf $SOURCETARBALL
 168     - rm -rf gromacs-*/$BUILD_DIR
 169     - tar -c --gzip -f $SOURCETARBALL gromacs-*/
 170     - mv $SOURCETARBALL ..
 171     - cd ..
 172     - rm -rf tempdir
 173     # Hack ends here
 174   artifacts:
 175     when: always
 176     expire_in: 1 week
 177     paths:
 178       - $BUILD_DIR/*log
 179       - gromacs*tar.gz
 180
 181 archive:build:
 182   extends:
 183     - .archive:build
 184     - .rules:nightly-not-for-release
 185   stage: nightly-build
 186   needs:
 187     - job: archive:configure:nightly
 188   variables:
 189     BUILD_DIR: build-package
 190
 191 archive:package:
 192   extends:
 193     - .archive:build
 194     - .rules:nightly-only-for-release
 195   stage: release-package
 196   needs:
 197     - job: archive:configure:release
 198   variables:
 199     BUILD_DIR: build-package
 200
 201 release-verify:
 202   image: biophysics/gcc-gromacs
 203   stage: release-verify
 204   extends:
 205     - .variables:default
 206     - .rules:nightly-only-for-release
 207   cache: {}
 208
 209   variables:
 210     KUBERNETES_CPU_LIMIT: 1
 211     KUBERNETES_CPU_REQUEST: 1
 212     KUBERNETES_MEMORY_REQUEST: 2Gi
 213   script:
 214     - VERSION=`cat version.json |
 215       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 216     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 217       VERSION=$VERSION-dev ;
 218       fi
 219     - REGTEST_COMPARE=`cat version.json |
 220       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 221     - SOURCENAME=gromacs-$VERSION
 222     - SOURCETARBALL=$SOURCENAME.tar.gz
 223     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 224     - REGTESTNAME=regressiontests-$VERSION
 225     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 226     - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'`
 227     - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM"
 228     - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM"
 229     - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE"
 230     - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then
 231       echo "Mismatch in regressiontest md5sums";
 232       exit 1;
 233       fi
 234   dependencies:
 235     - archive:package
 236     - regressiontests:package
 237     - prepare-release-version
 238
 239 archive:nightly-webpage:
 240   extends:
 241     - .webpage:build
 242     - .rules:nightly-not-for-release
 243   stage: nightly-deploy
 244   needs:
 245     - job: linkchecker
 246       artifacts: false
 247     - job: webpage:build
 248   variables:
 249     BUILD_DIR: build-docs
 250   script:
 251     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 252   artifacts:
 253     when: always
 254     paths:
 255       - webpage.tar.gz
 256
 257 archive:full-release:
 258   extends:
 259     - .webpage:build
 260     - .rules:nightly-only-for-release
 261   stage: release-deploy
 262   dependencies:
 263     - webpage:build:release
 264     - archive:package
 265     - regressiontests:package
 266   variables:
 267     BUILD_DIR: release-doc-builds
 268   script:
 269     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 270   artifacts:
 271     when: always
 272     paths:
 273       - webpage.tar.gz
 274       - gromacs-*tar.gz
 275       - regressiontests-*tar.gz
 276