Use CMake to propagate versions and hashes to gitlab jobs
[alexxy/gromacs.git] / admin / gitlab-ci / documentation.gitlab-ci.yml
index d808a57c8d277e6fa81452280f37adeb04a5c5f6..5018fdfc450b8d3c293a693ad951254f8f295d9e 100644 (file)
@@ -1,35 +1,25 @@
 # Build documentation and web page.
-.documentation-before-script-template:
-  variables:
-    EXTRA_INSTALLS: ""
-
-  before_script:
-    - echo $EXTRA_INSTALLS
-    - time apt-get -qq update
-    - time apt-get -qqy --no-install-suggests --no-install-recommends install
-      $EXTRA_INSTALLS
-    - mkdir -p ccache
-    - export CCACHE_BASEDIR=${PWD}
-    - export CCACHE_DIR=${PWD}/ccache
 
-.configure-docs-template:
-  # Built by admin/dockerfiles/ci-docs-clang
-  # TODO this should be organized more like the current documentation.py script
-  image: gromacs/ci-docs-clang:master
+.docs:configure:
+  extends:
+    - .variables:default
+    - .gromacs:base:configure
+    - .before_script:default
+  # TODO (#3480) this should be organized more like the current documentation.py script
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: configure-build
+  cache: {}
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
-    CMAKE_EXTRA_OPTIONS: ""
     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
-    CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
-
+    CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
   script:
     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
       mkdir $BUILD_DIR ;
       fi
     - cd $BUILD_DIR
+    # Running CMake with the venv activated should not be strictly necessary,
+    # but helps to find and cache self-consistent Python and Sphinx details
+    # without additional hinting. Once CMakeCache.txt exists, the later stages
+    # should work fine without reactivating the venv.
+    - source /root/venv/py3.7/bin/activate
     - cmake ..
       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
       $CMAKE_COMPILER_SCRIPT
     paths:
       - $BUILD_DIR
 
-.configure-docs-release-template:
-  # Built by admin/dockerfiles/ci-docs-clang
-  # TODO this should be organized more like the current documentation.py script
-  image: gromacs/ci-docs-clang:master
+.docs:configure:release:
+  extends:
+    - .variables:default
+    - .gromacs:base:configure
+    - .before_script:default
+    - .rules:nightly-only-for-release
+  # TODO (#3480) this should be organized more like the current documentation.py script
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   stage: release-configure
+  cache: {}
   variables:
     KUBERNETES_CPU_LIMIT: 1
     KUBERNETES_CPU_REQUEST: 1
-    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_MEMORY_REQUEST: 2Gi
     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
-    CMAKE_EXTRA_OPTIONS: ""
     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
-    CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF"
+    CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
     RELEASE_BUILD_DIR: release-doc-builds
     RELEASE_SOURCE: release-source-from-tarball
     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
-
-  only:
-    refs:
-      - web
-      - schedules
-      - triggers
-    variables:
-      - $GROMACS_RELEASE
   script:
     - if [[ $GROMACS_RELEASE == "true" ]] ; then
       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
     - echo $CMAKE_BUILD_TYPE_OPTIONS
     - echo $CMAKE_GMXAPI_OPTIONS
     - echo $REL_OPTION
-    - VERSION=`cat version.json |
-      python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
+    - VERSION=`sed -n -e '/GMX_VERSION_STRING\b/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
     - if [[ $GROMACS_RELEASE != "true" ]] ; then
       VERSION=$VERSION-dev ;
       fi
-    - REGTEST_COMPARE=`cat version.json |
-      python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
+    - REGTEST_COMPARE=`sed -n -e '/REGRESSIONTEST_MD5SUM/ s/.*"\(.*\)")/\1/p' VersionInfo-partial.cmake.cmakein`
     - SOURCENAME=gromacs-$VERSION
     - SOURCETARBALL=$SOURCENAME.tar.gz
     - REGTESTNAME=regressiontests-$VERSION
       mkdir $RELEASE_BUILD_DIR ;
       fi
     - cd $RELEASE_BUILD_DIR
+    - source /root/venv/py3.7/bin/activate
     - cmake ../$RELEASE_SOURCE/
       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
       $CMAKE_COMPILER_SCRIPT
       - $RELEASE_REGRESSIONTESTS
       - $RELEASE_SOURCE
 
-.configure-docs:
+docs:configure:
   extends:
-    - .gromacs:base:configure
-    - .configure-docs-template
-    - .documentation-before-script-template
-
-.configure-docs-release:
-  extends:
-    - .gromacs:base:configure
-    - .configure-docs-release-template
-    - .documentation-before-script-template
-  cache: {}
-
-configure-docs:
-  extends:
-    - .configure-docs
-  needs:
-    - job: simple-build
-      artifacts: false
+    - .docs:configure
+    - .rules:basic-push
   variables:
     BUILD_DIR: build-docs
-    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
-  except:
-    variables:
-      - $GROMACS_RELEASE
-
-configure-docs-gmxapi:
-  extends:
-    - .configure-docs
-    - .configure-gmxapi-template
-  needs:
-    - job: simple-build
-      artifacts: false
-  variables:
-    BUILD_DIR: build-docs-gmxapi
-    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
-  except:
-    variables:
-      - $GROMACS_RELEASE
+    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 
-.build-docs-binary-template:
-  # Built by admin/dockerfiles/ci-docs-clang
-  # TODO this should be organized more like the current documentation.py script
-  image: gromacs/ci-docs-clang:master
+.docs:build:
+  # TODO (#3480) this should be organized more like the current documentation.py script
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
   script:
     - cd $BUILD_DIR
     - cmake --build . --target gmx -- -j8
@@ -241,14 +198,40 @@ configure-docs-gmxapi:
     paths:
       - $BUILD_DIR
 
-.build-docs-webpage-template:
-  # Built by admin/dockerfiles/ci-docs-clang
-  # TODO this should be organized more like the current documentation.py script
-  image: gromacs/ci-docs-clang:master
+docs:build:
+  extends:
+    - .variables:default
+    - .gromacs:base:build
+    - .use-ccache
+    - .docs:build
+    - .before_script:default
+    - .rules:basic-push
+  stage: documentation
+  needs:
+    - job: docs:configure
+  variables:
+    BUILD_DIR: build-docs
+
+# The manual build is built separately so that errors in converting
+# Sphinx to LaTeX and compiling can always be found in the
+# all-output.txt file, while avoiding many thousands of lines of spam
+# from pdflatex for normal builds. This does reduce the available
+# parallel utilization, and so increases the build time.
+#
+# TODO why are the doxygen and sphinx log files disappearing
+# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
+.webpage:build:
+  extends:
+    - .variables:default
+    - .gromacs:base:build
+    - .before_script:default
+  # TODO (#3480) this should be organized more like the current documentation.py script
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-llvm-7-docs
+  cache: {}
   variables:
     KUBERNETES_CPU_LIMIT: 4
     KUBERNETES_CPU_REQUEST: 2
-    KUBERNETES_MEMORY_LIMIT: 4Gi
+    KUBERNETES_MEMORY_REQUEST: 4Gi
   script:
     - cd $BUILD_DIR
     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
@@ -275,172 +258,58 @@ configure-docs-gmxapi:
       - $BUILD_DIR/docs/doxygen/doxygen-full.log
       - $BUILD_DIR/docs/sphinx-html.log
 
-.build-docs-binary:
-  extends:
-    - .gromacs:base:build
-    - .use-ccache
-    - .variables:default
-    - .use-ccache
-    - .build-docs-binary-template
-    - .documentation-before-script-template
-
-.build-docs-webpage:
-  extends:
-    - .gromacs:base:build
-    - .variables:default
-    - .build-docs-webpage-template
-    - .documentation-before-script-template
-
-build-docs:
-  extends:
-    - .build-docs-binary
-  stage: documentation
-  needs:
-    - job: simple-build
-      artifacts: false
-    - job: configure-docs
-      artifacts: true
-  variables:
-    BUILD_DIR: build-docs
-  except:
-    variables:
-      - $GROMACS_RELEASE
-
-build-docs-gmxapi:
-  extends:
-    - .build-docs-binary
-    - .gmxapi-extra-installs
-  stage: documentation
-  needs:
-    - job: simple-build
-      artifacts: false
-    - job: configure-docs-gmxapi
-      artifacts: true
-  variables:
-    BUILD_DIR: build-docs-gmxapi
-  except:
-    variables:
-      - $GROMACS_RELEASE
-
-
-# The manual build is built separately so that errors in converting
-# Sphinx to LaTeX and compiling can always be found in the
-# all-output.txt file, while avoiding many thousands of lines of spam
-# from pdflatex for normal builds. This does reduce the available
-# parallel utilization, and so increases the build time.
-#
-# TODO why are the doxygen and sphinx log files disappearing
-# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
-webpage:
+webpage:build:
   extends:
-    - .build-docs-webpage
+    - .webpage:build
+    - .rules:basic-push
   stage: post-test
   needs:
-    - job: build-docs
-      artifacts: true
-    - job: run-check-source
+    - job: docs:build
+    - job: check-source
       artifacts: false
   variables:
     BUILD_DIR: build-docs
-  except:
-    variables:
-      - $GROMACS_RELEASE
 
-webpage-gmxapi:
+webpage:configure:
   extends:
-    - .build-docs-webpage
-    - .gmxapi-extra-installs
-  stage: post-test
-  needs:
-    - job: build-docs-gmxapi
-      artifacts: true
-    - job: run-check-source
-      artifacts: false
-  variables:
-    BUILD_DIR: build-docs-gmxapi
-  except:
-    variables:
-      - $GROMACS_RELEASE
-
-webpage-nightly:
-  extends:
-    - .build-docs-webpage
-    - .configure-gmxapi-template
-  stage: nightly-deploy
-  only:
-    - triggers
-    - schedules
-  except:
-    variables:
-      - $GROMACS_RELEASE
-  needs:
-    - job: simple-build
-      artifacts: false
-    - job: linkchecker
-      artifacts: false
-    - job: webpage-gmxapi
-      artifacts: true
-  variables:
-    BUILD_DIR: build-docs-gmxapi
-  script:
-    - tar czf webpage.tar.gz $BUILD_DIR/docs/html/
-  artifacts:
-    when: always
-    paths:
-      - webpage.tar.gz
-
-webpage-configure-release:
-  extends:
-    - .configure-docs-release
-    - .configure-gmxapi-template
+    - .docs:configure:release
+    - .rules:nightly-only-for-release
   cache: {}
-  only:
-    refs:
-      - web
-      - schedules
-      - triggers
-    variables:
-      - $GROMACS_RELEASE
+  tags:
+    - k8s-scilifelab
   dependencies:
-    - package-archive-release
-    - package-regressiontests-release
+    - archive:package
+    - regressiontests:package
     - prepare-release-version
   variables:
-    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on
+    CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 
 
-webpage-binary-release:
+webpage:dependencies:
   extends:
-    - .build-docs-binary
-    - .gmxapi-extra-installs
+    - .gromacs:base:build
+    - .variables:default
+    - .use-ccache
+    - .docs:build
+    - .before_script:default
+    - .rules:nightly-only-for-release
   stage: release-build
+  tags:
+    - k8s-scilifelab
   needs:
-    - job: webpage-configure-release
-      artifacts: true
+    - job: webpage:configure
   variables:
     BUILD_DIR: release-doc-builds
-  only:
-    refs:
-      - triggers
-      - web
-      - schedules
-    variables:
-      - $GROMACS_RELEASE
 
-webpage-build-release:
+webpage:build:release:
   extends:
-    - .build-docs-webpage
-    - .configure-gmxapi-template
-  dependencies:
-    - webpage-binary-release
-    - webpage-configure-release
+    - .webpage:build
+    - .rules:nightly-only-for-release
   stage: release-prepare-deploy
-  only:
-    refs:
-      - web
-      - schedules
-      - triggers
-    variables:
-      - $GROMACS_RELEASE
+  tags:
+    - k8s-scilifelab
+  dependencies:
+    - webpage:dependencies
+    - webpage:configure
   variables:
     BUILD_DIR: release-doc-builds