admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2
   3 .docs:configure:
   4   extends:
   5     - .variables:default
   6     - .gromacs:base:configure
   7     - .before_script:default
   8   # TODO (#3480) this should be organized more like the current documentation.py script
   9   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  10   stage: configure-build
  11   cache: {}
  12   variables:
  13     KUBERNETES_CPU_LIMIT: 1
  14     KUBERNETES_CPU_REQUEST: 1
  15     KUBERNETES_MEMORY_REQUEST: 2Gi
  16     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  17     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  18     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  19     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  20     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  21     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  22     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  23   script:
  24     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  25       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  26       else
  27       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  28       fi
  29     - echo $CMAKE_COMPILER_SCRIPT
  30     - echo $CMAKE_EXTRA_OPTIONS
  31     - echo $CMAKE_SIMD_OPTIONS
  32     - echo $CMAKE_GPU_OPTIONS
  33     - echo $CMAKE_MPI_OPTIONS
  34     - echo $CMAKE_PRECISION_OPTIONS
  35     - echo $CMAKE_BUILD_TYPE_OPTIONS
  36     - echo $CMAKE_GMXAPI_OPTIONS
  37     - echo $REL_OPTION
  38     - if [[ -d $BUILD_DIR ]] ; then
  39       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  40       else
  41       echo "Preparing new build directory" ;
  42       mkdir $BUILD_DIR ;
  43       fi
  44     - cd $BUILD_DIR
  45     # Running CMake with the venv activated should not be strictly necessary,
  46     # but helps to find and cache self-consistent Python and Sphinx details
  47     # without additional hinting. Once CMakeCache.txt exists, the later stages
  48     # should work fine without reactivating the venv.
  49     - source /root/venv/py3.7/bin/activate
  50     # TODO(#4092) Remove `pip install pybind11` once CI images are updated.
  51     - /root/venv/py3.7/bin/pip install pybind11
  52     - cmake ..
  53       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  54       $CMAKE_COMPILER_SCRIPT
  55       $CMAKE_EXTRA_OPTIONS
  56       $CMAKE_SIMD_OPTIONS
  57       $CMAKE_MPI_OPTIONS
  58       $CMAKE_PRECISION_OPTIONS
  59       $CMAKE_BUILD_TYPE_OPTIONS
  60       $CMAKE_GPU_OPTIONS
  61       $CMAKE_GMXAPI_OPTIONS
  62       $REL_OPTION
  63       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  64       2>&1 | tee cmakeLog.log
  65     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  66     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  67     - cd ..
  68   artifacts:
  69     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  70     when: always
  71     expire_in: 1 week
  72     paths:
  73       - $BUILD_DIR
  74
  75 .docs:configure:release:
  76   extends:
  77     - .variables:default
  78     - .gromacs:base:configure
  79     - .before_script:default
  80     - .rules:nightly-only-for-release
  81   # TODO (#3480) this should be organized more like the current documentation.py script
  82   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  83   stage: release-configure
  84   cache: {}
  85   variables:
  86     KUBERNETES_CPU_LIMIT: 1
  87     KUBERNETES_CPU_REQUEST: 1
  88     KUBERNETES_MEMORY_REQUEST: 2Gi
  89     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  90     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  91     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  92     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  93     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  94     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  95     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  96     RELEASE_BUILD_DIR: release-doc-builds
  97     RELEASE_SOURCE: release-source-from-tarball
  98     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
  99   script:
 100     - if [[ $GROMACS_RELEASE == "true" ]] ; then
 101       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 102       else
 103       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 104       fi
 105     - echo $CMAKE_COMPILER_SCRIPT
 106     - echo $CMAKE_EXTRA_OPTIONS
 107     - echo $CMAKE_SIMD_OPTIONS
 108     - echo $CMAKE_GPU_OPTIONS
 109     - echo $CMAKE_MPI_OPTIONS
 110     - echo $CMAKE_PRECISION_OPTIONS
 111     - echo $CMAKE_BUILD_TYPE_OPTIONS
 112     - echo $CMAKE_GMXAPI_OPTIONS
 113     - echo $REL_OPTION
 114     - VERSION=`cat version.json |
 115       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 116     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 117       VERSION=$VERSION-dev ;
 118       fi
 119     - REGTEST_COMPARE=`cat version.json |
 120       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 121     - SOURCENAME=gromacs-$VERSION
 122     - SOURCETARBALL=$SOURCENAME.tar.gz
 123     - REGTESTNAME=regressiontests-$VERSION
 124     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 125     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 126     - RELEASE_TARBALLS=release-tarballs
 127     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 128     - mv $SOURCETARBALL $RELEASE_TARBALLS
 129     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 130     - cd $RELEASE_TARBALLS
 131     # We rename the source and regressiontest directories
 132     # to have names for them that don't change for different versions.
 133     - tar -xf $SOURCETARBALL
 134     - mv $SOURCENAME ../$RELEASE_SOURCE
 135     - tar -xf $REGTESTTARBALL
 136     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 137     - rm $SOURCETARBALL $REGTESTTARBALL
 138     - cd ..
 139     - echo $CMAKE_COMPILER_SCRIPT
 140     - echo $CMAKE_EXTRA_OPTIONS
 141     - echo $CMAKE_SIMD_OPTIONS
 142     - echo $CMAKE_GPU_OPTIONS
 143     - echo $CMAKE_MPI_OPTIONS
 144     - echo $CMAKE_PRECISION_OPTIONS
 145     - echo $CMAKE_BUILD_TYPE_OPTIONS
 146     - echo $CMAKE_GMXAPI_OPTIONS
 147     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 148       echo "Cleaning up build directory" ;
 149       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 150       else
 151       echo "Preparing new build directory" ;
 152       mkdir $RELEASE_BUILD_DIR ;
 153       fi
 154     - cd $RELEASE_BUILD_DIR
 155     - source /root/venv/py3.7/bin/activate
 156     # TODO(#4092) Remove `pip install pybind11` once CI images are updated.
 157     - /root/venv/py3.7/bin/pip install pybind11
 158     - cmake ../$RELEASE_SOURCE/
 159       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 160       $CMAKE_COMPILER_SCRIPT
 161       $CMAKE_EXTRA_OPTIONS
 162       $CMAKE_SIMD_OPTIONS
 163       $CMAKE_MPI_OPTIONS
 164       $CMAKE_PRECISION_OPTIONS
 165       $CMAKE_BUILD_TYPE_OPTIONS
 166       $CMAKE_GPU_OPTIONS
 167       $CMAKE_GMXAPI_OPTIONS
 168       $REL_OPTION
 169       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 170       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 171       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 172       2>&1 | tee cmakeLog.log
 173     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 174     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 175     - cd ..
 176   artifacts:
 177     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 178     when: always
 179     expire_in: 1 week
 180     paths:
 181       - $RELEASE_BUILD_DIR
 182       - $RELEASE_REGRESSIONTESTS
 183       - $RELEASE_SOURCE
 184
 185 docs:configure:
 186   extends:
 187     - .docs:configure
 188     - .rules:basic-push
 189   variables:
 190     BUILD_DIR: build-docs
 191     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 192
 193 .docs:build:
 194   # TODO (#3480) this should be organized more like the current documentation.py script
 195   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 196   script:
 197     - cd $BUILD_DIR
 198     # TODO(#4092) Remove `pip install pybind11` once CI images are updated.
 199     - /root/venv/py3.7/bin/pip install pybind11
 200     - cmake --build . --target gmx -- -j8
 201     - cd ..
 202   artifacts:
 203     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 204     when: always
 205     expire_in: 1 week
 206     paths:
 207       - $BUILD_DIR
 208
 209 docs:build:
 210   extends:
 211     - .variables:default
 212     - .gromacs:base:build
 213     - .use-ccache
 214     - .docs:build
 215     - .before_script:default
 216     - .rules:basic-push
 217   stage: documentation
 218   needs:
 219     - job: docs:configure
 220   variables:
 221     BUILD_DIR: build-docs
 222
 223 # The manual build is built separately so that errors in converting
 224 # Sphinx to LaTeX and compiling can always be found in the
 225 # all-output.txt file, while avoiding many thousands of lines of spam
 226 # from pdflatex for normal builds. This does reduce the available
 227 # parallel utilization, and so increases the build time.
 228 #
 229 # TODO why are the doxygen and sphinx log files disappearing
 230 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 231 .webpage:build:
 232   extends:
 233     - .variables:default
 234     - .gromacs:base:build
 235     - .before_script:default
 236   # TODO (#3480) this should be organized more like the current documentation.py script
 237   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 238   cache: {}
 239   variables:
 240     KUBERNETES_CPU_LIMIT: 4
 241     KUBERNETES_CPU_REQUEST: 2
 242     KUBERNETES_MEMORY_REQUEST: 4Gi
 243   script:
 244     - cd $BUILD_DIR
 245     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 246       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 247     # TODO(#4092) Remove `pip install pybind11` once CI images are updated.
 248     - /root/venv/py3.7/bin/pip install pybind11
 249     - cmake --build . --target webpage -- -j4
 250     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 251     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 252     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 253     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 254     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 255     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 256     - cd ..
 257   artifacts:
 258     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 259     when: always
 260     expire_in: 1 week
 261     paths:
 262       - $BUILD_DIR/*.log
 263       - $BUILD_DIR/docs/html
 264       - $BUILD_DIR/docs/manual/gromacs.log
 265       - $BUILD_DIR/docs/manual/all-output.log
 266       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 267       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 268       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 269       - $BUILD_DIR/docs/sphinx-html.log
 270
 271 webpage:build:
 272   extends:
 273     - .webpage:build
 274     - .rules:basic-push
 275   stage: post-test
 276   needs:
 277     - job: docs:build
 278     - job: check-source
 279       artifacts: false
 280   variables:
 281     BUILD_DIR: build-docs
 282
 283 webpage:configure:
 284   extends:
 285     - .docs:configure:release
 286     - .rules:nightly-only-for-release
 287   cache: {}
 288   tags:
 289     - k8s-scilifelab
 290   dependencies:
 291     - archive:package
 292     - regressiontests:package
 293     - prepare-release-version
 294   variables:
 295     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 296
 297
 298 webpage:dependencies:
 299   extends:
 300     - .gromacs:base:build
 301     - .variables:default
 302     - .use-ccache
 303     - .docs:build
 304     - .before_script:default
 305     - .rules:nightly-only-for-release
 306   stage: release-build
 307   tags:
 308     - k8s-scilifelab
 309   needs:
 310     - job: webpage:configure
 311   variables:
 312     BUILD_DIR: release-doc-builds
 313
 314 webpage:build:release:
 315   extends:
 316     - .webpage:build
 317     - .rules:nightly-only-for-release
 318   stage: release-prepare-deploy
 319   tags:
 320     - k8s-scilifelab
 321   dependencies:
 322     - webpage:dependencies
 323     - webpage:configure
 324   variables:
 325     BUILD_DIR: release-doc-builds