admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2 .documentation-before-script-template:
   3   variables:
   4     EXTRA_INSTALLS: ""
   5
   6   before_script:
   7     - echo $EXTRA_INSTALLS
   8     - time apt-get -qq update
   9     - time apt-get -qqy --no-install-suggests --no-install-recommends install
  10       $EXTRA_INSTALLS
  11     - mkdir -p ccache
  12     - export CCACHE_BASEDIR=${PWD}
  13     - export CCACHE_DIR=${PWD}/ccache
  14
  15 .configure-docs-template:
  16   # Built by admin/dockerfiles/ci-docs-clang
  17   # TODO this should be organized more like the current documentation.py script
  18   image: gromacs/ci-docs-clang:master
  19   stage: configure-build
  20   variables:
  21     KUBERNETES_CPU_LIMIT: 1
  22     KUBERNETES_CPU_REQUEST: 1
  23     KUBERNETES_MEMORY_LIMIT: 2Gi
  24     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  25     CMAKE_EXTRA_OPTIONS: ""
  26     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  27     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  28     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  29     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  30     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  31     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
  32
  33   script:
  34     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  35       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  36       else
  37       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  38       fi
  39     - echo $CMAKE_COMPILER_SCRIPT
  40     - echo $CMAKE_EXTRA_OPTIONS
  41     - echo $CMAKE_SIMD_OPTIONS
  42     - echo $CMAKE_GPU_OPTIONS
  43     - echo $CMAKE_MPI_OPTIONS
  44     - echo $CMAKE_PRECISION_OPTIONS
  45     - echo $CMAKE_BUILD_TYPE_OPTIONS
  46     - echo $CMAKE_GMXAPI_OPTIONS
  47     - echo $REL_OPTION
  48     - if [[ -d $BUILD_DIR ]] ; then
  49       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  50       else
  51       echo "Preparing new build directory" ;
  52       mkdir $BUILD_DIR ;
  53       fi
  54     - cd $BUILD_DIR
  55     - cmake ..
  56       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  57       $CMAKE_COMPILER_SCRIPT
  58       $CMAKE_EXTRA_OPTIONS
  59       $CMAKE_SIMD_OPTIONS
  60       $CMAKE_MPI_OPTIONS
  61       $CMAKE_PRECISION_OPTIONS
  62       $CMAKE_BUILD_TYPE_OPTIONS
  63       $CMAKE_GPU_OPTIONS
  64       $CMAKE_GMXAPI_OPTIONS
  65       $REL_OPTION
  66       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  67       2>&1 | tee cmakeLog.log
  68     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  69     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  70     - cd ..
  71   artifacts:
  72     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  73     when: always
  74     expire_in: 1 week
  75     paths:
  76       - $BUILD_DIR
  77
  78 .configure-docs-release-template:
  79   # Built by admin/dockerfiles/ci-docs-clang
  80   # TODO this should be organized more like the current documentation.py script
  81   image: gromacs/ci-docs-clang:master
  82   stage: release-configure
  83   variables:
  84     KUBERNETES_CPU_LIMIT: 1
  85     KUBERNETES_CPU_REQUEST: 1
  86     KUBERNETES_MEMORY_LIMIT: 2Gi
  87     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  88     CMAKE_EXTRA_OPTIONS: ""
  89     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  90     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  91     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  92     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  93     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  94     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
  95     RELEASE_BUILD_DIR: release-doc-builds
  96     RELEASE_SOURCE: release-source-from-tarball
  97     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
  98
  99   only:
 100     refs:
 101       - web
 102       - schedules
 103       - triggers
 104     variables:
 105       - $GROMACS_RELEASE
 106   script:
 107     - if [[ $GROMACS_RELEASE == "true" ]] ; then
 108       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 109       else
 110       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 111       fi
 112     - echo $CMAKE_COMPILER_SCRIPT
 113     - echo $CMAKE_EXTRA_OPTIONS
 114     - echo $CMAKE_SIMD_OPTIONS
 115     - echo $CMAKE_GPU_OPTIONS
 116     - echo $CMAKE_MPI_OPTIONS
 117     - echo $CMAKE_PRECISION_OPTIONS
 118     - echo $CMAKE_BUILD_TYPE_OPTIONS
 119     - echo $CMAKE_GMXAPI_OPTIONS
 120     - echo $REL_OPTION
 121     - VERSION=`cat version.json |
 122       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 123     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 124       VERSION=$VERSION-dev ;
 125       fi
 126     - REGTEST_COMPARE=`cat version.json |
 127       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 128     - SOURCENAME=gromacs-$VERSION
 129     - SOURCETARBALL=$SOURCENAME.tar.gz
 130     - REGTESTNAME=regressiontests-$VERSION
 131     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 132     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 133     - RELEASE_TARBALLS=release-tarballs
 134     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 135     - mv $SOURCETARBALL $RELEASE_TARBALLS
 136     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 137     - cd $RELEASE_TARBALLS
 138     # We rename the source and regressiontest directories
 139     # to have names for them that don't change for different versions.
 140     - tar -xf $SOURCETARBALL
 141     - mv $SOURCENAME ../$RELEASE_SOURCE
 142     - tar -xf $REGTESTTARBALL
 143     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 144     - rm $SOURCETARBALL $REGTESTTARBALL
 145     - cd ..
 146     - echo $CMAKE_COMPILER_SCRIPT
 147     - echo $CMAKE_EXTRA_OPTIONS
 148     - echo $CMAKE_SIMD_OPTIONS
 149     - echo $CMAKE_GPU_OPTIONS
 150     - echo $CMAKE_MPI_OPTIONS
 151     - echo $CMAKE_PRECISION_OPTIONS
 152     - echo $CMAKE_BUILD_TYPE_OPTIONS
 153     - echo $CMAKE_GMXAPI_OPTIONS
 154     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 155       echo "Cleaning up build directory" ;
 156       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 157       else
 158       echo "Preparing new build directory" ;
 159       mkdir $RELEASE_BUILD_DIR ;
 160       fi
 161     - cd $RELEASE_BUILD_DIR
 162     - cmake ../$RELEASE_SOURCE/
 163       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 164       $CMAKE_COMPILER_SCRIPT
 165       $CMAKE_EXTRA_OPTIONS
 166       $CMAKE_SIMD_OPTIONS
 167       $CMAKE_MPI_OPTIONS
 168       $CMAKE_PRECISION_OPTIONS
 169       $CMAKE_BUILD_TYPE_OPTIONS
 170       $CMAKE_GPU_OPTIONS
 171       $CMAKE_GMXAPI_OPTIONS
 172       $REL_OPTION
 173       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 174       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 175       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 176       2>&1 | tee cmakeLog.log
 177     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 178     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 179     - cd ..
 180   artifacts:
 181     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 182     when: always
 183     expire_in: 1 week
 184     paths:
 185       - $RELEASE_BUILD_DIR
 186       - $RELEASE_REGRESSIONTESTS
 187       - $RELEASE_SOURCE
 188
 189 .configure-docs:
 190   extends:
 191     - .gromacs:base:configure
 192     - .configure-docs-template
 193     - .documentation-before-script-template
 194
 195 .configure-docs-release:
 196   extends:
 197     - .gromacs:base:configure
 198     - .configure-docs-release-template
 199     - .documentation-before-script-template
 200   cache: {}
 201
 202 configure-docs:
 203   extends:
 204     - .configure-docs
 205   needs:
 206     - job: simple-build
 207       artifacts: false
 208   variables:
 209     BUILD_DIR: build-docs
 210     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 211   except:
 212     variables:
 213       - $GROMACS_RELEASE
 214
 215 configure-docs-gmxapi:
 216   extends:
 217     - .configure-docs
 218     - .configure-gmxapi-template
 219   needs:
 220     - job: simple-build
 221       artifacts: false
 222   variables:
 223     BUILD_DIR: build-docs-gmxapi
 224     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 225   except:
 226     variables:
 227       - $GROMACS_RELEASE
 228
 229 .build-docs-binary-template:
 230   # Built by admin/dockerfiles/ci-docs-clang
 231   # TODO this should be organized more like the current documentation.py script
 232   image: gromacs/ci-docs-clang:master
 233   script:
 234     - cd $BUILD_DIR
 235     - cmake --build . --target gmx -- -j8
 236     - cd ..
 237   artifacts:
 238     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 239     when: always
 240     expire_in: 1 week
 241     paths:
 242       - $BUILD_DIR
 243
 244 .build-docs-webpage-template:
 245   # Built by admin/dockerfiles/ci-docs-clang
 246   # TODO this should be organized more like the current documentation.py script
 247   image: gromacs/ci-docs-clang:master
 248   variables:
 249     KUBERNETES_CPU_LIMIT: 4
 250     KUBERNETES_CPU_REQUEST: 2
 251     KUBERNETES_MEMORY_LIMIT: 4Gi
 252   script:
 253     - cd $BUILD_DIR
 254     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 255       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 256     - cmake --build . --target webpage -- -j4
 257     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 258     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 259     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 260     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 261     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 262     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 263     - cd ..
 264   artifacts:
 265     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 266     when: always
 267     expire_in: 1 week
 268     paths:
 269       - $BUILD_DIR/*.log
 270       - $BUILD_DIR/docs/html
 271       - $BUILD_DIR/docs/manual/gromacs.log
 272       - $BUILD_DIR/docs/manual/all-output.log
 273       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 274       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 275       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 276       - $BUILD_DIR/docs/sphinx-html.log
 277
 278 .build-docs-binary:
 279   extends:
 280     - .gromacs:base:build
 281     - .use-ccache
 282     - .variables:default
 283     - .use-ccache
 284     - .build-docs-binary-template
 285     - .documentation-before-script-template
 286
 287 .build-docs-webpage:
 288   extends:
 289     - .gromacs:base:build
 290     - .variables:default
 291     - .build-docs-webpage-template
 292     - .documentation-before-script-template
 293
 294 build-docs:
 295   extends:
 296     - .build-docs-binary
 297   stage: documentation
 298   needs:
 299     - job: simple-build
 300       artifacts: false
 301     - job: configure-docs
 302       artifacts: true
 303   variables:
 304     BUILD_DIR: build-docs
 305   except:
 306     variables:
 307       - $GROMACS_RELEASE
 308
 309 build-docs-gmxapi:
 310   extends:
 311     - .build-docs-binary
 312     - .gmxapi-extra-installs
 313   stage: documentation
 314   needs:
 315     - job: simple-build
 316       artifacts: false
 317     - job: configure-docs-gmxapi
 318       artifacts: true
 319   variables:
 320     BUILD_DIR: build-docs-gmxapi
 321   except:
 322     variables:
 323       - $GROMACS_RELEASE
 324
 325
 326 # The manual build is built separately so that errors in converting
 327 # Sphinx to LaTeX and compiling can always be found in the
 328 # all-output.txt file, while avoiding many thousands of lines of spam
 329 # from pdflatex for normal builds. This does reduce the available
 330 # parallel utilization, and so increases the build time.
 331 #
 332 # TODO why are the doxygen and sphinx log files disappearing
 333 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 334 webpage:
 335   extends:
 336     - .build-docs-webpage
 337   stage: post-test
 338   needs:
 339     - job: build-docs
 340       artifacts: true
 341     - job: run-check-source
 342       artifacts: false
 343   variables:
 344     BUILD_DIR: build-docs
 345   except:
 346     variables:
 347       - $GROMACS_RELEASE
 348
 349 webpage-gmxapi:
 350   extends:
 351     - .build-docs-webpage
 352     - .gmxapi-extra-installs
 353   stage: post-test
 354   needs:
 355     - job: build-docs-gmxapi
 356       artifacts: true
 357     - job: run-check-source
 358       artifacts: false
 359   variables:
 360     BUILD_DIR: build-docs-gmxapi
 361   except:
 362     variables:
 363       - $GROMACS_RELEASE
 364
 365 webpage-nightly:
 366   extends:
 367     - .build-docs-webpage
 368     - .configure-gmxapi-template
 369   stage: nightly-deploy
 370   only:
 371     - triggers
 372     - schedules
 373   except:
 374     variables:
 375       - $GROMACS_RELEASE
 376   needs:
 377     - job: simple-build
 378       artifacts: false
 379     - job: linkchecker
 380       artifacts: false
 381     - job: webpage-gmxapi
 382       artifacts: true
 383   variables:
 384     BUILD_DIR: build-docs-gmxapi
 385   script:
 386     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 387   artifacts:
 388     when: always
 389     paths:
 390       - webpage.tar.gz
 391
 392 webpage-configure-release:
 393   extends:
 394     - .configure-docs-release
 395     - .configure-gmxapi-template
 396   cache: {}
 397   only:
 398     refs:
 399       - web
 400       - schedules
 401       - triggers
 402     variables:
 403       - $GROMACS_RELEASE
 404   dependencies:
 405     - package-archive-release
 406     - package-regressiontests-release
 407     - prepare-release-version
 408   variables:
 409     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 410
 411
 412 webpage-binary-release:
 413   extends:
 414     - .build-docs-binary
 415     - .gmxapi-extra-installs
 416   stage: release-build
 417   needs:
 418     - job: webpage-configure-release
 419       artifacts: true
 420   variables:
 421     BUILD_DIR: release-doc-builds
 422   only:
 423     refs:
 424       - triggers
 425       - web
 426       - schedules
 427     variables:
 428       - $GROMACS_RELEASE
 429
 430 webpage-build-release:
 431   extends:
 432     - .build-docs-webpage
 433     - .configure-gmxapi-template
 434   dependencies:
 435     - webpage-binary-release
 436     - webpage-configure-release
 437   stage: release-prepare-deploy
 438   only:
 439     refs:
 440       - web
 441       - schedules
 442       - triggers
 443     variables:
 444       - $GROMACS_RELEASE
 445   variables:
 446     BUILD_DIR: release-doc-builds