admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2
   3 .docs:configure:
   4   extends:
   5     - .variables:default
   6     - .gromacs:base:configure
   7     - .before_script:default
   8   # TODO (#3480) this should be organized more like the current documentation.py script
   9   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  10   stage: configure-build
  11   cache: {}
  12   variables:
  13     KUBERNETES_CPU_LIMIT: 1
  14     KUBERNETES_CPU_REQUEST: 1
  15     KUBERNETES_MEMORY_REQUEST: 2Gi
  16     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  17     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  18     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  19     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  20     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  21     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  22     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  23   script:
  24     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  25       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  26       else
  27       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  28       fi
  29     - echo $CMAKE_COMPILER_SCRIPT
  30     - echo $CMAKE_EXTRA_OPTIONS
  31     - echo $CMAKE_SIMD_OPTIONS
  32     - echo $CMAKE_GPU_OPTIONS
  33     - echo $CMAKE_MPI_OPTIONS
  34     - echo $CMAKE_PRECISION_OPTIONS
  35     - echo $CMAKE_BUILD_TYPE_OPTIONS
  36     - echo $CMAKE_GMXAPI_OPTIONS
  37     - echo $REL_OPTION
  38     - if [[ -d $BUILD_DIR ]] ; then
  39       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  40       else
  41       echo "Preparing new build directory" ;
  42       mkdir $BUILD_DIR ;
  43       fi
  44     - cd $BUILD_DIR
  45     # Running CMake with the venv activated should not be strictly necessary,
  46     # but helps to find and cache self-consistent Python and Sphinx details
  47     # without additional hinting. Once CMakeCache.txt exists, the later stages
  48     # should work fine without reactivating the venv.
  49     - source /root/venv/py3.7/bin/activate
  50     - cmake ..
  51       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  52       $CMAKE_COMPILER_SCRIPT
  53       $CMAKE_EXTRA_OPTIONS
  54       $CMAKE_SIMD_OPTIONS
  55       $CMAKE_MPI_OPTIONS
  56       $CMAKE_PRECISION_OPTIONS
  57       $CMAKE_BUILD_TYPE_OPTIONS
  58       $CMAKE_GPU_OPTIONS
  59       $CMAKE_GMXAPI_OPTIONS
  60       $REL_OPTION
  61       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  62       2>&1 | tee cmakeLog.log
  63     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  64     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  65     - cd ..
  66   artifacts:
  67     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  68     when: always
  69     expire_in: 1 week
  70     paths:
  71       - $BUILD_DIR
  72
  73 .docs:configure:release:
  74   extends:
  75     - .variables:default
  76     - .gromacs:base:configure
  77     - .before_script:default
  78     - .rules:nightly-only-for-release
  79   # TODO (#3480) this should be organized more like the current documentation.py script
  80   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
  81   stage: release-configure
  82   cache: {}
  83   variables:
  84     KUBERNETES_CPU_LIMIT: 1
  85     KUBERNETES_CPU_REQUEST: 1
  86     KUBERNETES_MEMORY_REQUEST: 2Gi
  87     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  88     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  89     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  90     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  91     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  92     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  93     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  94     RELEASE_BUILD_DIR: release-doc-builds
  95     RELEASE_SOURCE: release-source-from-tarball
  96     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
  97   script:
  98     - if [[ $GROMACS_RELEASE == "true" ]] ; then
  99       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 100       else
 101       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 102       fi
 103     - echo $CMAKE_COMPILER_SCRIPT
 104     - echo $CMAKE_EXTRA_OPTIONS
 105     - echo $CMAKE_SIMD_OPTIONS
 106     - echo $CMAKE_GPU_OPTIONS
 107     - echo $CMAKE_MPI_OPTIONS
 108     - echo $CMAKE_PRECISION_OPTIONS
 109     - echo $CMAKE_BUILD_TYPE_OPTIONS
 110     - echo $CMAKE_GMXAPI_OPTIONS
 111     - echo $REL_OPTION
 112     - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
 113     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 114       VERSION=$VERSION-dev ;
 115       fi
 116     - REGTEST_COMPARE=`sed -n -e '/REGRESSIONTEST_MD5SUM/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
 117     - SOURCENAME=gromacs-$VERSION
 118     - SOURCETARBALL=$SOURCENAME.tar.gz
 119     - REGTESTNAME=regressiontests-$VERSION
 120     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 121     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 122     - RELEASE_TARBALLS=release-tarballs
 123     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 124     - mv $SOURCETARBALL $RELEASE_TARBALLS
 125     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 126     - cd $RELEASE_TARBALLS
 127     # We rename the source and regressiontest directories
 128     # to have names for them that don't change for different versions.
 129     - tar -xf $SOURCETARBALL
 130     - mv $SOURCENAME ../$RELEASE_SOURCE
 131     - tar -xf $REGTESTTARBALL
 132     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 133     - rm $SOURCETARBALL $REGTESTTARBALL
 134     - cd ..
 135     - echo $CMAKE_COMPILER_SCRIPT
 136     - echo $CMAKE_EXTRA_OPTIONS
 137     - echo $CMAKE_SIMD_OPTIONS
 138     - echo $CMAKE_GPU_OPTIONS
 139     - echo $CMAKE_MPI_OPTIONS
 140     - echo $CMAKE_PRECISION_OPTIONS
 141     - echo $CMAKE_BUILD_TYPE_OPTIONS
 142     - echo $CMAKE_GMXAPI_OPTIONS
 143     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 144       echo "Cleaning up build directory" ;
 145       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 146       else
 147       echo "Preparing new build directory" ;
 148       mkdir $RELEASE_BUILD_DIR ;
 149       fi
 150     - cd $RELEASE_BUILD_DIR
 151     - source /root/venv/py3.7/bin/activate
 152     - cmake ../$RELEASE_SOURCE/
 153       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 154       $CMAKE_COMPILER_SCRIPT
 155       $CMAKE_EXTRA_OPTIONS
 156       $CMAKE_SIMD_OPTIONS
 157       $CMAKE_MPI_OPTIONS
 158       $CMAKE_PRECISION_OPTIONS
 159       $CMAKE_BUILD_TYPE_OPTIONS
 160       $CMAKE_GPU_OPTIONS
 161       $CMAKE_GMXAPI_OPTIONS
 162       $REL_OPTION
 163       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 164       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 165       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 166       2>&1 | tee cmakeLog.log
 167     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 168     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 169     - cd ..
 170   artifacts:
 171     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 172     when: always
 173     expire_in: 1 week
 174     paths:
 175       - $RELEASE_BUILD_DIR
 176       - $RELEASE_REGRESSIONTESTS
 177       - $RELEASE_SOURCE
 178
 179 docs:configure:
 180   extends:
 181     - .docs:configure
 182     - .rules:basic-push
 183   variables:
 184     BUILD_DIR: build-docs
 185     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 186
 187 .docs:build:
 188   # TODO (#3480) this should be organized more like the current documentation.py script
 189   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 190   script:
 191     - cd $BUILD_DIR
 192     - cmake --build . --target gmx -- -j8
 193     - cd ..
 194   artifacts:
 195     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 196     when: always
 197     expire_in: 1 week
 198     paths:
 199       - $BUILD_DIR
 200
 201 docs:build:
 202   extends:
 203     - .variables:default
 204     - .gromacs:base:build
 205     - .use-ccache
 206     - .docs:build
 207     - .before_script:default
 208     - .rules:basic-push
 209   stage: documentation
 210   needs:
 211     - job: docs:configure
 212   variables:
 213     BUILD_DIR: build-docs
 214
 215 # The manual build is built separately so that errors in converting
 216 # Sphinx to LaTeX and compiling can always be found in the
 217 # all-output.txt file, while avoiding many thousands of lines of spam
 218 # from pdflatex for normal builds. This does reduce the available
 219 # parallel utilization, and so increases the build time.
 220 #
 221 # TODO why are the doxygen and sphinx log files disappearing
 222 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 223 .webpage:build:
 224   extends:
 225     - .variables:default
 226     - .gromacs:base:build
 227     - .before_script:default
 228   # TODO (#3480) this should be organized more like the current documentation.py script
 229   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
 230   cache: {}
 231   variables:
 232     KUBERNETES_CPU_LIMIT: 4
 233     KUBERNETES_CPU_REQUEST: 2
 234     KUBERNETES_MEMORY_REQUEST: 4Gi
 235   script:
 236     - cd $BUILD_DIR
 237     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 238       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 239     - cmake --build . --target webpage -- -j4
 240     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 241     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 242     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 243     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 244     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 245     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 246     - cd ..
 247   artifacts:
 248     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 249     when: always
 250     expire_in: 1 week
 251     paths:
 252       - $BUILD_DIR/*.log
 253       - $BUILD_DIR/docs/html
 254       - $BUILD_DIR/docs/manual/gromacs.log
 255       - $BUILD_DIR/docs/manual/all-output.log
 256       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 257       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 258       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 259       - $BUILD_DIR/docs/sphinx-html.log
 260
 261 webpage:build:
 262   extends:
 263     - .webpage:build
 264     - .rules:basic-push
 265   stage: post-test
 266   needs:
 267     - job: docs:build
 268     - job: check-source
 269       artifacts: false
 270   variables:
 271     BUILD_DIR: build-docs
 272
 273 webpage:configure:
 274   extends:
 275     - .docs:configure:release
 276     - .rules:nightly-only-for-release
 277   cache: {}
 278   tags:
 279     - k8s-scilifelab
 280   dependencies:
 281     - archive:package
 282     - regressiontests:package
 283     - prepare-release-version
 284   variables:
 285     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 286
 287
 288 webpage:dependencies:
 289   extends:
 290     - .gromacs:base:build
 291     - .variables:default
 292     - .use-ccache
 293     - .docs:build
 294     - .before_script:default
 295     - .rules:nightly-only-for-release
 296   stage: release-build
 297   tags:
 298     - k8s-scilifelab
 299   needs:
 300     - job: webpage:configure
 301   variables:
 302     BUILD_DIR: release-doc-builds
 303
 304 webpage:build:release:
 305   extends:
 306     - .webpage:build
 307     - .rules:nightly-only-for-release
 308   stage: release-prepare-deploy
 309   tags:
 310     - k8s-scilifelab
 311   dependencies:
 312     - webpage:dependencies
 313     - webpage:configure
 314   variables:
 315     BUILD_DIR: release-doc-builds