admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2 .documentation-before-script-template:
   3   before_script:
   4     - echo $EXTRA_INSTALLS
   5     - time apt-get -qq update
   6     - time apt-get -qqy --no-install-suggests --no-install-recommends install
   7       $EXTRA_INSTALLS
   8     - mkdir -p ccache
   9     - export CCACHE_BASEDIR=${PWD}
  10     - export CCACHE_DIR=${PWD}/ccache
  11
  12 .configure-docs-template:
  13   # Built by admin/dockerfiles/ci-docs-clang
  14   # TODO this should be organized more like the current documentation.py script
  15   image: gromacs/ci-docs-clang:master
  16   stage: configure-build
  17   extends:
  18     - .variables:default
  19   only:
  20     refs:
  21       - web
  22       - pushes
  23       - schedules
  24       - merge_requests
  25   variables:
  26     KUBERNETES_CPU_LIMIT: 1
  27     KUBERNETES_CPU_REQUEST: 1
  28     KUBERNETES_MEMORY_LIMIT: 2Gi
  29     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  30     CMAKE_EXTRA_OPTIONS: ""
  31     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  32     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  33     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  34     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  35     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  36   script:
  37     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  38       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  39       else
  40       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  41       fi
  42     - echo $CMAKE_COMPILER_SCRIPT
  43     - echo $CMAKE_EXTRA_OPTIONS
  44     - echo $CMAKE_SIMD_OPTIONS
  45     - echo $CMAKE_GPU_OPTIONS
  46     - echo $CMAKE_MPI_OPTIONS
  47     - echo $CMAKE_PRECISION_OPTIONS
  48     - echo $CMAKE_BUILD_TYPE_OPTIONS
  49     - echo $CMAKE_GMXAPI_OPTIONS
  50     - echo $REL_OPTION
  51     - if [[ -d $BUILD_DIR ]] ; then
  52       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  53       else
  54       echo "Preparing new build directory" ;
  55       mkdir $BUILD_DIR ;
  56       fi
  57     - cd $BUILD_DIR
  58     - cmake ..
  59       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  60       $CMAKE_COMPILER_SCRIPT
  61       $CMAKE_EXTRA_OPTIONS
  62       $CMAKE_SIMD_OPTIONS
  63       $CMAKE_MPI_OPTIONS
  64       $CMAKE_PRECISION_OPTIONS
  65       $CMAKE_BUILD_TYPE_OPTIONS
  66       $CMAKE_GPU_OPTIONS
  67       $CMAKE_GMXAPI_OPTIONS
  68       $REL_OPTION
  69       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  70       2>&1 | tee cmakeLog.log
  71     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  72     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  73     - cd ..
  74   artifacts:
  75     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  76     when: always
  77     expire_in: 1 week
  78     paths:
  79       - $BUILD_DIR
  80
  81 .configure-docs-release-template:
  82   # Built by admin/dockerfiles/ci-docs-clang
  83   # TODO this should be organized more like the current documentation.py script
  84   image: gromacs/ci-docs-clang:master
  85   stage: release-configure
  86   extends:
  87     - .variables:default
  88   variables:
  89     KUBERNETES_CPU_LIMIT: 1
  90     KUBERNETES_CPU_REQUEST: 1
  91     KUBERNETES_MEMORY_LIMIT: 2Gi
  92     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  93     CMAKE_EXTRA_OPTIONS: ""
  94     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  95     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  96     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  97     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  98     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  99     RELEASE_BUILD_DIR: release-doc-builds
 100     RELEASE_SOURCE: release-source-from-tarball
 101     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
 102
 103   only:
 104     refs:
 105       - web
 106       - schedules
 107       - triggers
 108     variables:
 109       - $GROMACS_RELEASE
 110   script:
 111     - if [[ $GROMACS_RELEASE == "true" ]] ; then
 112       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 113       else
 114       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 115       fi
 116     - echo $CMAKE_COMPILER_SCRIPT
 117     - echo $CMAKE_EXTRA_OPTIONS
 118     - echo $CMAKE_SIMD_OPTIONS
 119     - echo $CMAKE_GPU_OPTIONS
 120     - echo $CMAKE_MPI_OPTIONS
 121     - echo $CMAKE_PRECISION_OPTIONS
 122     - echo $CMAKE_BUILD_TYPE_OPTIONS
 123     - echo $CMAKE_GMXAPI_OPTIONS
 124     - echo $REL_OPTION
 125     - VERSION=`cat version.json |
 126       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 127     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 128       VERSION=$VERSION-dev ;
 129       fi
 130     - REGTEST_COMPARE=`cat version.json |
 131       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 132     - SOURCENAME=gromacs-$VERSION
 133     - SOURCETARBALL=$SOURCENAME.tar.gz
 134     - REGTESTNAME=regressiontests-$VERSION
 135     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 136     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 137     - RELEASE_TARBALLS=release-tarballs
 138     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 139     - mv $SOURCETARBALL $RELEASE_TARBALLS
 140     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 141     - cd $RELEASE_TARBALLS
 142     # We rename the source and regressiontest directories
 143     # to have names for them that don't change for different versions.
 144     - tar -xf $SOURCETARBALL
 145     - mv $SOURCENAME ../$RELEASE_SOURCE
 146     - tar -xf $REGTESTTARBALL
 147     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 148     - rm $SOURCETARBALL $REGTESTTARBALL
 149     - cd ..
 150     - echo $CMAKE_COMPILER_SCRIPT
 151     - echo $CMAKE_EXTRA_OPTIONS
 152     - echo $CMAKE_SIMD_OPTIONS
 153     - echo $CMAKE_GPU_OPTIONS
 154     - echo $CMAKE_MPI_OPTIONS
 155     - echo $CMAKE_PRECISION_OPTIONS
 156     - echo $CMAKE_BUILD_TYPE_OPTIONS
 157     - echo $CMAKE_GMXAPI_OPTIONS
 158     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 159       echo "Cleaning up build directory" ;
 160       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 161       else
 162       echo "Preparing new build directory" ;
 163       mkdir $RELEASE_BUILD_DIR ;
 164       fi
 165     - cd $RELEASE_BUILD_DIR
 166     - cmake ../$RELEASE_SOURCE/
 167       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 168       $CMAKE_COMPILER_SCRIPT
 169       $CMAKE_EXTRA_OPTIONS
 170       $CMAKE_SIMD_OPTIONS
 171       $CMAKE_MPI_OPTIONS
 172       $CMAKE_PRECISION_OPTIONS
 173       $CMAKE_BUILD_TYPE_OPTIONS
 174       $CMAKE_GPU_OPTIONS
 175       $CMAKE_GMXAPI_OPTIONS
 176       $REL_OPTION
 177       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 178       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 179       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 180       2>&1 | tee cmakeLog.log
 181     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 182     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 183     - cd ..
 184   artifacts:
 185     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 186     when: always
 187     expire_in: 1 week
 188     paths:
 189       - $RELEASE_BUILD_DIR
 190       - $RELEASE_REGRESSIONTESTS
 191       - $RELEASE_SOURCE
 192
 193 .configure-docs:
 194   extends:
 195     - .gromacs:base:configure
 196     - .configure-docs-template
 197     - .documentation-before-script-template
 198
 199 .configure-docs-release:
 200   extends:
 201     - .gromacs:base:configure
 202     - .configure-docs-release-template
 203     - .documentation-before-script-template
 204   cache: {}
 205
 206 configure-docs:
 207   extends:
 208     - .configure-docs
 209   needs:
 210     - job: simple-build
 211       artifacts: false
 212   variables:
 213     BUILD_DIR: build-docs
 214     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 215   except:
 216     variables:
 217       - $GROMACS_RELEASE
 218
 219 .build-docs-binary-template:
 220   # Built by admin/dockerfiles/ci-docs-clang
 221   # TODO this should be organized more like the current documentation.py script
 222   image: gromacs/ci-docs-clang:master
 223   only:
 224     refs:
 225       - web
 226       - pushes
 227       - schedules
 228       - merge_requests
 229   script:
 230     - cd $BUILD_DIR
 231     - cmake --build . --target gmx -- -j8
 232     - cd ..
 233   artifacts:
 234     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 235     when: always
 236     expire_in: 1 week
 237     paths:
 238       - $BUILD_DIR
 239
 240 .build-docs-webpage-template:
 241   # Built by admin/dockerfiles/ci-docs-clang
 242   # TODO this should be organized more like the current documentation.py script
 243   image: gromacs/ci-docs-clang:master
 244   only:
 245     refs:
 246       - web
 247       - pushes
 248       - schedules
 249       - merge_requests
 250   variables:
 251     KUBERNETES_CPU_LIMIT: 4
 252     KUBERNETES_CPU_REQUEST: 2
 253     KUBERNETES_MEMORY_LIMIT: 4Gi
 254   script:
 255     - cd $BUILD_DIR
 256     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 257       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 258     - cmake --build . --target webpage -- -j4
 259     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 260     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 261     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 262     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 263     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 264     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 265     - cd ..
 266   artifacts:
 267     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 268     when: always
 269     expire_in: 1 week
 270     paths:
 271       - $BUILD_DIR/*.log
 272       - $BUILD_DIR/docs/html
 273       - $BUILD_DIR/docs/manual/gromacs.log
 274       - $BUILD_DIR/docs/manual/all-output.log
 275       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 276       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 277       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 278       - $BUILD_DIR/docs/sphinx-html.log
 279
 280 .build-docs-binary:
 281   extends:
 282     - .gromacs:base:build
 283     - .use-ccache
 284     - .variables:default
 285     - .use-ccache
 286     - .build-docs-binary-template
 287     - .documentation-before-script-template
 288
 289 .build-docs-webpage:
 290   extends:
 291     - .gromacs:base:build
 292     - .variables:default
 293     - .build-docs-webpage-template
 294     - .documentation-before-script-template
 295
 296 build-docs:
 297   extends:
 298     - .build-docs-binary
 299   stage: documentation
 300   needs:
 301     - job: simple-build
 302       artifacts: false
 303     - job: configure-docs
 304       artifacts: true
 305   variables:
 306     BUILD_DIR: build-docs
 307   except:
 308     variables:
 309       - $GROMACS_RELEASE
 310
 311 # The manual build is built separately so that errors in converting
 312 # Sphinx to LaTeX and compiling can always be found in the
 313 # all-output.txt file, while avoiding many thousands of lines of spam
 314 # from pdflatex for normal builds. This does reduce the available
 315 # parallel utilization, and so increases the build time.
 316 #
 317 # TODO why are the doxygen and sphinx log files disappearing
 318 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 319 webpage:
 320   extends:
 321     - .build-docs-webpage
 322   stage: post-test
 323   needs:
 324     - job: build-docs
 325       artifacts: true
 326     - job: run-check-source
 327       artifacts: false
 328   variables:
 329     BUILD_DIR: build-docs
 330   except:
 331     variables:
 332       - $GROMACS_RELEASE
 333
 334 webpage-nightly:
 335   extends:
 336     - .build-docs-webpage
 337   stage: nightly-deploy
 338   only:
 339     - web
 340     - triggers
 341     - schedules
 342   except:
 343     variables:
 344       - $GROMACS_RELEASE
 345   needs:
 346     - job: simple-build
 347       artifacts: false
 348     - job: linkchecker
 349       artifacts: false
 350     - job: webpage
 351       artifacts: true
 352   variables:
 353     BUILD_DIR: build-docs
 354   script:
 355     - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
 356   artifacts:
 357     when: always
 358     paths:
 359       - webpage.tar.gz
 360
 361 webpage-configure-release:
 362   extends:
 363     - .configure-docs-release
 364   cache: {}
 365   only:
 366     refs:
 367       - web
 368       - schedules
 369       - triggers
 370     variables:
 371       - $GROMACS_RELEASE
 372   dependencies:
 373     - package-archive-release
 374     - package-regressiontests-release
 375     - prepare-release-version
 376   variables:
 377     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 378
 379
 380 webpage-binary-release:
 381   extends:
 382     - .build-docs-binary
 383   stage: release-build
 384   needs:
 385     - job: webpage-configure-release
 386       artifacts: true
 387   variables:
 388     BUILD_DIR: release-doc-builds
 389   only:
 390     refs:
 391       - triggers
 392       - web
 393       - schedules
 394     variables:
 395       - $GROMACS_RELEASE
 396
 397 webpage-build-release:
 398   extends:
 399     - .build-docs-webpage
 400   dependencies:
 401     - webpage-binary-release
 402     - webpage-configure-release
 403   stage: release-prepare-deploy
 404   only:
 405     refs:
 406       - web
 407       - schedules
 408       - triggers
 409     variables:
 410       - $GROMACS_RELEASE
 411   variables:
 412     BUILD_DIR: release-doc-builds