admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2 .documentation-before-script-template:
   3   variables:
   4     EXTRA_INSTALLS: ""
   5
   6   before_script:
   7     - echo $EXTRA_INSTALLS
   8     - time apt-get -qq update
   9     - time apt-get -qqy --no-install-suggests --no-install-recommends install
  10       $EXTRA_INSTALLS
  11     - mkdir -p ccache
  12     - export CCACHE_BASEDIR=${PWD}
  13     - export CCACHE_DIR=${PWD}/ccache
  14
  15 .configure-docs-template:
  16   # Built by admin/dockerfiles/ci-docs-clang
  17   # TODO this should be organized more like the current documentation.py script
  18   image: gromacs/ci-docs-clang:master
  19   stage: configure-build
  20   only:
  21     refs:
  22       - web
  23       - pushes
  24       - schedules
  25       - merge_requests
  26   variables:
  27     KUBERNETES_CPU_LIMIT: 1
  28     KUBERNETES_CPU_REQUEST: 1
  29     KUBERNETES_MEMORY_LIMIT: 2Gi
  30     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  31     CMAKE_EXTRA_OPTIONS: ""
  32     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  33     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  34     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  35     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  36     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  37     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
  38   script:
  39     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  40       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  41       else
  42       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  43       fi
  44     - echo $CMAKE_COMPILER_SCRIPT
  45     - echo $CMAKE_EXTRA_OPTIONS
  46     - echo $CMAKE_SIMD_OPTIONS
  47     - echo $CMAKE_GPU_OPTIONS
  48     - echo $CMAKE_MPI_OPTIONS
  49     - echo $CMAKE_PRECISION_OPTIONS
  50     - echo $CMAKE_BUILD_TYPE_OPTIONS
  51     - echo $CMAKE_GMXAPI_OPTIONS
  52     - echo $REL_OPTION
  53     - if [[ -d $BUILD_DIR ]] ; then
  54       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  55       else
  56       echo "Preparing new build directory" ;
  57       mkdir $BUILD_DIR ;
  58       fi
  59     - cd $BUILD_DIR
  60     - cmake ..
  61       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  62       $CMAKE_COMPILER_SCRIPT
  63       $CMAKE_EXTRA_OPTIONS
  64       $CMAKE_SIMD_OPTIONS
  65       $CMAKE_MPI_OPTIONS
  66       $CMAKE_PRECISION_OPTIONS
  67       $CMAKE_BUILD_TYPE_OPTIONS
  68       $CMAKE_GPU_OPTIONS
  69       $CMAKE_GMXAPI_OPTIONS
  70       $REL_OPTION
  71       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  72       2>&1 | tee cmakeLog.log
  73     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  74     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  75     - cd ..
  76   artifacts:
  77     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  78     when: always
  79     expire_in: 1 week
  80     paths:
  81       - $BUILD_DIR
  82
  83 .configure-docs-release-template:
  84   # Built by admin/dockerfiles/ci-docs-clang
  85   # TODO this should be organized more like the current documentation.py script
  86   image: gromacs/ci-docs-clang:master
  87   stage: release-configure
  88   variables:
  89     KUBERNETES_CPU_LIMIT: 1
  90     KUBERNETES_CPU_REQUEST: 1
  91     KUBERNETES_MEMORY_LIMIT: 2Gi
  92     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  93     CMAKE_EXTRA_OPTIONS: ""
  94     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  95     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  96     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  97     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  98     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  99     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
 100     RELEASE_BUILD_DIR: release-doc-builds
 101     RELEASE_SOURCE: release-source-from-tarball
 102     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
 103
 104   only:
 105     refs:
 106       - web
 107       - schedules
 108       - triggers
 109     variables:
 110       - $GROMACS_RELEASE
 111   script:
 112     - if [[ $GROMACS_RELEASE == "true" ]] ; then
 113       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 114       else
 115       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 116       fi
 117     - echo $CMAKE_COMPILER_SCRIPT
 118     - echo $CMAKE_EXTRA_OPTIONS
 119     - echo $CMAKE_SIMD_OPTIONS
 120     - echo $CMAKE_GPU_OPTIONS
 121     - echo $CMAKE_MPI_OPTIONS
 122     - echo $CMAKE_PRECISION_OPTIONS
 123     - echo $CMAKE_BUILD_TYPE_OPTIONS
 124     - echo $CMAKE_GMXAPI_OPTIONS
 125     - echo $REL_OPTION
 126     - VERSION=`cat version.json |
 127       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 128     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 129       VERSION=$VERSION-dev ;
 130       fi
 131     - REGTEST_COMPARE=`cat version.json |
 132       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 133     - SOURCENAME=gromacs-$VERSION
 134     - SOURCETARBALL=$SOURCENAME.tar.gz
 135     - REGTESTNAME=regressiontests-$VERSION
 136     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 137     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 138     - RELEASE_TARBALLS=release-tarballs
 139     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 140     - mv $SOURCETARBALL $RELEASE_TARBALLS
 141     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 142     - cd $RELEASE_TARBALLS
 143     # We rename the source and regressiontest directories
 144     # to have names for them that don't change for different versions.
 145     - tar -xf $SOURCETARBALL
 146     - mv $SOURCENAME ../$RELEASE_SOURCE
 147     - tar -xf $REGTESTTARBALL
 148     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 149     - rm $SOURCETARBALL $REGTESTTARBALL
 150     - cd ..
 151     - echo $CMAKE_COMPILER_SCRIPT
 152     - echo $CMAKE_EXTRA_OPTIONS
 153     - echo $CMAKE_SIMD_OPTIONS
 154     - echo $CMAKE_GPU_OPTIONS
 155     - echo $CMAKE_MPI_OPTIONS
 156     - echo $CMAKE_PRECISION_OPTIONS
 157     - echo $CMAKE_BUILD_TYPE_OPTIONS
 158     - echo $CMAKE_GMXAPI_OPTIONS
 159     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 160       echo "Cleaning up build directory" ;
 161       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 162       else
 163       echo "Preparing new build directory" ;
 164       mkdir $RELEASE_BUILD_DIR ;
 165       fi
 166     - cd $RELEASE_BUILD_DIR
 167     - cmake ../$RELEASE_SOURCE/
 168       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 169       $CMAKE_COMPILER_SCRIPT
 170       $CMAKE_EXTRA_OPTIONS
 171       $CMAKE_SIMD_OPTIONS
 172       $CMAKE_MPI_OPTIONS
 173       $CMAKE_PRECISION_OPTIONS
 174       $CMAKE_BUILD_TYPE_OPTIONS
 175       $CMAKE_GPU_OPTIONS
 176       $CMAKE_GMXAPI_OPTIONS
 177       $REL_OPTION
 178       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 179       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 180       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 181       2>&1 | tee cmakeLog.log
 182     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 183     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 184     - cd ..
 185   artifacts:
 186     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 187     when: always
 188     expire_in: 1 week
 189     paths:
 190       - $RELEASE_BUILD_DIR
 191       - $RELEASE_REGRESSIONTESTS
 192       - $RELEASE_SOURCE
 193
 194 .configure-docs:
 195   extends:
 196     - .gromacs:base:configure
 197     - .configure-docs-template
 198     - .documentation-before-script-template
 199
 200 .configure-docs-release:
 201   extends:
 202     - .gromacs:base:configure
 203     - .configure-docs-release-template
 204     - .documentation-before-script-template
 205   cache: {}
 206
 207 configure-docs:
 208   extends:
 209     - .configure-docs
 210   needs:
 211     - job: simple-build
 212       artifacts: false
 213   variables:
 214     BUILD_DIR: build-docs
 215     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 216   except:
 217     variables:
 218       - $GROMACS_RELEASE
 219
 220 configure-docs-gmxapi:
 221   extends:
 222     - .configure-docs
 223     - .configure-gmxapi-template
 224   needs:
 225     - job: simple-build
 226       artifacts: false
 227   variables:
 228     BUILD_DIR: build-docs-gmxapi
 229     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 230   except:
 231     variables:
 232       - $GROMACS_RELEASE
 233
 234 .build-docs-binary-template:
 235   # Built by admin/dockerfiles/ci-docs-clang
 236   # TODO this should be organized more like the current documentation.py script
 237   image: gromacs/ci-docs-clang:master
 238   only:
 239     refs:
 240       - web
 241       - pushes
 242       - schedules
 243       - merge_requests
 244   script:
 245     - cd $BUILD_DIR
 246     - cmake --build . --target gmx -- -j8
 247     - cd ..
 248   artifacts:
 249     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 250     when: always
 251     expire_in: 1 week
 252     paths:
 253       - $BUILD_DIR
 254
 255 .build-docs-webpage-template:
 256   # Built by admin/dockerfiles/ci-docs-clang
 257   # TODO this should be organized more like the current documentation.py script
 258   image: gromacs/ci-docs-clang:master
 259   only:
 260     refs:
 261       - web
 262       - pushes
 263       - schedules
 264       - merge_requests
 265   variables:
 266     KUBERNETES_CPU_LIMIT: 4
 267     KUBERNETES_CPU_REQUEST: 2
 268     KUBERNETES_MEMORY_LIMIT: 4Gi
 269   script:
 270     - cd $BUILD_DIR
 271     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 272       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 273     - cmake --build . --target webpage -- -j4
 274     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 275     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 276     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 277     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 278     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 279     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 280     - cd ..
 281   artifacts:
 282     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 283     when: always
 284     expire_in: 1 week
 285     paths:
 286       - $BUILD_DIR/*.log
 287       - $BUILD_DIR/docs/html
 288       - $BUILD_DIR/docs/manual/gromacs.log
 289       - $BUILD_DIR/docs/manual/all-output.log
 290       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 291       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 292       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 293       - $BUILD_DIR/docs/sphinx-html.log
 294
 295 .build-docs-binary:
 296   extends:
 297     - .gromacs:base:build
 298     - .use-ccache
 299     - .variables:default
 300     - .use-ccache
 301     - .build-docs-binary-template
 302     - .documentation-before-script-template
 303
 304 .build-docs-webpage:
 305   extends:
 306     - .gromacs:base:build
 307     - .variables:default
 308     - .build-docs-webpage-template
 309     - .documentation-before-script-template
 310
 311 build-docs:
 312   extends:
 313     - .build-docs-binary
 314   stage: documentation
 315   needs:
 316     - job: simple-build
 317       artifacts: false
 318     - job: configure-docs
 319       artifacts: true
 320   variables:
 321     BUILD_DIR: build-docs
 322   except:
 323     variables:
 324       - $GROMACS_RELEASE
 325
 326 build-docs-gmxapi:
 327   extends:
 328     - .build-docs-binary
 329     - .gmxapi-extra-installs
 330   stage: documentation
 331   needs:
 332     - job: simple-build
 333       artifacts: false
 334     - job: configure-docs-gmxapi
 335       artifacts: true
 336   variables:
 337     BUILD_DIR: build-docs-gmxapi
 338   except:
 339     variables:
 340       - $GROMACS_RELEASE
 341
 342
 343 # The manual build is built separately so that errors in converting
 344 # Sphinx to LaTeX and compiling can always be found in the
 345 # all-output.txt file, while avoiding many thousands of lines of spam
 346 # from pdflatex for normal builds. This does reduce the available
 347 # parallel utilization, and so increases the build time.
 348 #
 349 # TODO why are the doxygen and sphinx log files disappearing
 350 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 351 webpage:
 352   extends:
 353     - .build-docs-webpage
 354   stage: post-test
 355   needs:
 356     - job: build-docs
 357       artifacts: true
 358     - job: run-check-source
 359       artifacts: false
 360   variables:
 361     BUILD_DIR: build-docs
 362   except:
 363     variables:
 364       - $GROMACS_RELEASE
 365
 366 webpage-gmxapi:
 367   extends:
 368     - .build-docs-webpage
 369     - .gmxapi-extra-installs
 370   stage: post-test
 371   needs:
 372     - job: build-docs-gmxapi
 373       artifacts: true
 374     - job: run-check-source
 375       artifacts: false
 376   variables:
 377     BUILD_DIR: build-docs-gmxapi
 378   except:
 379     variables:
 380       - $GROMACS_RELEASE
 381
 382 webpage-nightly:
 383   extends:
 384     - .build-docs-webpage
 385     - .configure-gmxapi-template
 386   stage: nightly-deploy
 387   only:
 388     - web
 389     - triggers
 390     - schedules
 391   except:
 392     variables:
 393       - $GROMACS_RELEASE
 394   needs:
 395     - job: simple-build
 396       artifacts: false
 397     - job: linkchecker
 398       artifacts: false
 399     - job: webpage-gmxapi
 400       artifacts: true
 401   variables:
 402     BUILD_DIR: build-docs-gmxapi
 403   script:
 404     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 405   artifacts:
 406     when: always
 407     paths:
 408       - webpage.tar.gz
 409
 410 webpage-configure-release:
 411   extends:
 412     - .configure-docs-release
 413     - .configure-gmxapi-template
 414   cache: {}
 415   only:
 416     refs:
 417       - web
 418       - schedules
 419       - triggers
 420     variables:
 421       - $GROMACS_RELEASE
 422   dependencies:
 423     - package-archive-release
 424     - package-regressiontests-release
 425     - prepare-release-version
 426   variables:
 427     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 428
 429
 430 webpage-binary-release:
 431   extends:
 432     - .build-docs-binary
 433     - .gmxapi-extra-installs
 434   stage: release-build
 435   needs:
 436     - job: webpage-configure-release
 437       artifacts: true
 438   variables:
 439     BUILD_DIR: release-doc-builds
 440   only:
 441     refs:
 442       - triggers
 443       - web
 444       - schedules
 445     variables:
 446       - $GROMACS_RELEASE
 447
 448 webpage-build-release:
 449   extends:
 450     - .build-docs-webpage
 451     - .configure-gmxapi-template
 452   dependencies:
 453     - webpage-binary-release
 454     - webpage-configure-release
 455   stage: release-prepare-deploy
 456   only:
 457     refs:
 458       - web
 459       - schedules
 460       - triggers
 461     variables:
 462       - $GROMACS_RELEASE
 463   variables:
 464     BUILD_DIR: release-doc-builds