admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2
   3 .docs:configure:
   4   # Built by admin/dockerfiles/ci-docs-clang
   5   # TODO this should be organized more like the current documentation.py script
   6   image: gromacs/ci-docs-clang:master
   7   stage: configure-build
   8   extends:
   9     - .gromacs:base:configure
  10     - .variables:default
  11     - .before_script:default
  12   only:
  13     refs:
  14       - web
  15       - pushes
  16       - schedules
  17       - merge_requests
  18   variables:
  19     KUBERNETES_CPU_LIMIT: 1
  20     KUBERNETES_CPU_REQUEST: 1
  21     KUBERNETES_MEMORY_LIMIT: 2Gi
  22     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  23     CMAKE_EXTRA_OPTIONS: ""
  24     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  25     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  26     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  27     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  28     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  29   script:
  30     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  31       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  32       else
  33       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  34       fi
  35     - echo $CMAKE_COMPILER_SCRIPT
  36     - echo $CMAKE_EXTRA_OPTIONS
  37     - echo $CMAKE_SIMD_OPTIONS
  38     - echo $CMAKE_GPU_OPTIONS
  39     - echo $CMAKE_MPI_OPTIONS
  40     - echo $CMAKE_PRECISION_OPTIONS
  41     - echo $CMAKE_BUILD_TYPE_OPTIONS
  42     - echo $CMAKE_GMXAPI_OPTIONS
  43     - echo $REL_OPTION
  44     - if [[ -d $BUILD_DIR ]] ; then
  45       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  46       else
  47       echo "Preparing new build directory" ;
  48       mkdir $BUILD_DIR ;
  49       fi
  50     - cd $BUILD_DIR
  51     - cmake ..
  52       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  53       $CMAKE_COMPILER_SCRIPT
  54       $CMAKE_EXTRA_OPTIONS
  55       $CMAKE_SIMD_OPTIONS
  56       $CMAKE_MPI_OPTIONS
  57       $CMAKE_PRECISION_OPTIONS
  58       $CMAKE_BUILD_TYPE_OPTIONS
  59       $CMAKE_GPU_OPTIONS
  60       $CMAKE_GMXAPI_OPTIONS
  61       $REL_OPTION
  62       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  63       2>&1 | tee cmakeLog.log
  64     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  65     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  66     - cd ..
  67   artifacts:
  68     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  69     when: always
  70     expire_in: 1 week
  71     paths:
  72       - $BUILD_DIR
  73
  74 .docs:configure:release:
  75   # Built by admin/dockerfiles/ci-docs-clang
  76   # TODO this should be organized more like the current documentation.py script
  77   image: gromacs/ci-docs-clang:master
  78   stage: release-configure
  79   extends:
  80     - .gromacs:base:configure
  81     - .variables:default
  82     - .before_script:default
  83   cache: {}
  84   variables:
  85     KUBERNETES_CPU_LIMIT: 1
  86     KUBERNETES_CPU_REQUEST: 1
  87     KUBERNETES_MEMORY_LIMIT: 2Gi
  88     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  89     CMAKE_EXTRA_OPTIONS: ""
  90     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  91     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  92     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  93     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  94     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  95     RELEASE_BUILD_DIR: release-doc-builds
  96     RELEASE_SOURCE: release-source-from-tarball
  97     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
  98
  99   only:
 100     refs:
 101       - web
 102       - schedules
 103       - triggers
 104     variables:
 105       - $GROMACS_RELEASE
 106   script:
 107     - if [[ $GROMACS_RELEASE == "true" ]] ; then
 108       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
 109       else
 110       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
 111       fi
 112     - echo $CMAKE_COMPILER_SCRIPT
 113     - echo $CMAKE_EXTRA_OPTIONS
 114     - echo $CMAKE_SIMD_OPTIONS
 115     - echo $CMAKE_GPU_OPTIONS
 116     - echo $CMAKE_MPI_OPTIONS
 117     - echo $CMAKE_PRECISION_OPTIONS
 118     - echo $CMAKE_BUILD_TYPE_OPTIONS
 119     - echo $CMAKE_GMXAPI_OPTIONS
 120     - echo $REL_OPTION
 121     - VERSION=`cat version.json |
 122       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 123     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 124       VERSION=$VERSION-dev ;
 125       fi
 126     - REGTEST_COMPARE=`cat version.json |
 127       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 128     - SOURCENAME=gromacs-$VERSION
 129     - SOURCETARBALL=$SOURCENAME.tar.gz
 130     - REGTESTNAME=regressiontests-$VERSION
 131     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 132     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 133     - RELEASE_TARBALLS=release-tarballs
 134     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 135     - mv $SOURCETARBALL $RELEASE_TARBALLS
 136     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 137     - cd $RELEASE_TARBALLS
 138     # We rename the source and regressiontest directories
 139     # to have names for them that don't change for different versions.
 140     - tar -xf $SOURCETARBALL
 141     - mv $SOURCENAME ../$RELEASE_SOURCE
 142     - tar -xf $REGTESTTARBALL
 143     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 144     - rm $SOURCETARBALL $REGTESTTARBALL
 145     - cd ..
 146     - echo $CMAKE_COMPILER_SCRIPT
 147     - echo $CMAKE_EXTRA_OPTIONS
 148     - echo $CMAKE_SIMD_OPTIONS
 149     - echo $CMAKE_GPU_OPTIONS
 150     - echo $CMAKE_MPI_OPTIONS
 151     - echo $CMAKE_PRECISION_OPTIONS
 152     - echo $CMAKE_BUILD_TYPE_OPTIONS
 153     - echo $CMAKE_GMXAPI_OPTIONS
 154     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 155       echo "Cleaning up build directory" ;
 156       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 157       else
 158       echo "Preparing new build directory" ;
 159       mkdir $RELEASE_BUILD_DIR ;
 160       fi
 161     - cd $RELEASE_BUILD_DIR
 162     - cmake ../$RELEASE_SOURCE/
 163       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 164       $CMAKE_COMPILER_SCRIPT
 165       $CMAKE_EXTRA_OPTIONS
 166       $CMAKE_SIMD_OPTIONS
 167       $CMAKE_MPI_OPTIONS
 168       $CMAKE_PRECISION_OPTIONS
 169       $CMAKE_BUILD_TYPE_OPTIONS
 170       $CMAKE_GPU_OPTIONS
 171       $CMAKE_GMXAPI_OPTIONS
 172       $REL_OPTION
 173       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 174       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 175       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 176       2>&1 | tee cmakeLog.log
 177     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 178     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 179     - cd ..
 180   artifacts:
 181     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 182     when: always
 183     expire_in: 1 week
 184     paths:
 185       - $RELEASE_BUILD_DIR
 186       - $RELEASE_REGRESSIONTESTS
 187       - $RELEASE_SOURCE
 188
 189 docs:configure:
 190   extends:
 191     - .docs:configure
 192   needs:
 193     - job: simple-build
 194       artifacts: false
 195   variables:
 196     BUILD_DIR: build-docs
 197     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 198   except:
 199     variables:
 200       - $GROMACS_RELEASE
 201
 202 .docs:build:
 203   # Built by admin/dockerfiles/ci-docs-clang
 204   # TODO this should be organized more like the current documentation.py script
 205   image: gromacs/ci-docs-clang:master
 206   only:
 207     refs:
 208       - web
 209       - pushes
 210       - schedules
 211       - merge_requests
 212   script:
 213     - cd $BUILD_DIR
 214     - cmake --build . --target gmx -- -j8
 215     - cd ..
 216   artifacts:
 217     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 218     when: always
 219     expire_in: 1 week
 220     paths:
 221       - $BUILD_DIR
 222
 223 docs:build:
 224   extends:
 225     - .gromacs:base:build
 226     - .variables:default
 227     - .use-ccache
 228     - .docs:build
 229     - .before_script:default
 230   stage: documentation
 231   needs:
 232     - job: simple-build
 233       artifacts: false
 234     - job: docs:configure
 235       artifacts: true
 236   variables:
 237     BUILD_DIR: build-docs
 238   except:
 239     variables:
 240       - $GROMACS_RELEASE
 241
 242 # The manual build is built separately so that errors in converting
 243 # Sphinx to LaTeX and compiling can always be found in the
 244 # all-output.txt file, while avoiding many thousands of lines of spam
 245 # from pdflatex for normal builds. This does reduce the available
 246 # parallel utilization, and so increases the build time.
 247 #
 248 # TODO why are the doxygen and sphinx log files disappearing
 249 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 250 .webpage:build:
 251   extends:
 252     - .gromacs:base:build
 253     - .variables:default
 254     - .before_script:default
 255   # Built by admin/dockerfiles/ci-docs-clang
 256   # TODO this should be organized more like the current documentation.py script
 257   image: gromacs/ci-docs-clang:master
 258   only:
 259     refs:
 260       - web
 261       - pushes
 262       - schedules
 263       - merge_requests
 264   variables:
 265     KUBERNETES_CPU_LIMIT: 4
 266     KUBERNETES_CPU_REQUEST: 2
 267     KUBERNETES_MEMORY_LIMIT: 4Gi
 268   script:
 269     - cd $BUILD_DIR
 270     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 271       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 272     - cmake --build . --target webpage -- -j4
 273     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 274     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 275     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 276     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 277     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 278     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 279     - cd ..
 280   artifacts:
 281     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 282     when: always
 283     expire_in: 1 week
 284     paths:
 285       - $BUILD_DIR/*.log
 286       - $BUILD_DIR/docs/html
 287       - $BUILD_DIR/docs/manual/gromacs.log
 288       - $BUILD_DIR/docs/manual/all-output.log
 289       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 290       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 291       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 292       - $BUILD_DIR/docs/sphinx-html.log
 293
 294 webpage:build:
 295   extends:
 296     - .webpage:build
 297   stage: post-test
 298   needs:
 299     - job: docs:build
 300       artifacts: true
 301     - job: check-source
 302       artifacts: false
 303   variables:
 304     BUILD_DIR: build-docs
 305   except:
 306     variables:
 307       - $GROMACS_RELEASE
 308
 309 webpage:configure:
 310   extends:
 311     - .docs:configure:release
 312   cache: {}
 313   only:
 314     refs:
 315       - web
 316       - schedules
 317       - triggers
 318     variables:
 319       - $GROMACS_RELEASE
 320   dependencies:
 321     - archive:package
 322     - regressiontests:package
 323     - prepare-release-version
 324   variables:
 325     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
 326
 327
 328 webpage:dependencies:
 329   extends:
 330     - .gromacs:base:build
 331     - .variables:default
 332     - .use-ccache
 333     - .docs:build
 334     - .before_script:default
 335   stage: release-build
 336   needs:
 337     - job: webpage:configure
 338       artifacts: true
 339   variables:
 340     BUILD_DIR: release-doc-builds
 341   only:
 342     refs:
 343       - triggers
 344       - web
 345       - schedules
 346     variables:
 347       - $GROMACS_RELEASE
 348
 349 webpage:build:release:
 350   extends:
 351     - .webpage:build
 352   dependencies:
 353     - webpage:dependencies
 354     - webpage:configure
 355   stage: release-prepare-deploy
 356   only:
 357     refs:
 358       - web
 359       - schedules
 360       - triggers
 361     variables:
 362       - $GROMACS_RELEASE
 363   variables:
 364     BUILD_DIR: release-doc-builds