From 76bc730c44753e2bc4b24d97c7eaed209dd8e160 Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Sat, 21 Mar 2020 15:58:56 +0300 Subject: [PATCH] Remove redundant jobs. Remove distinction between jobs that do and do not stage a build of the gmxapi Python package in the build tree for API breakage testing and full documentation builds. Set a default CMake flag of "-DGMX_PYTHON_PACKAGE=ON" and install some Python package dependencies with the default EXTRA_INSTALLS. The extra installs will go away in the near future with updated Docker images. Removes (up to) 12 jobs from the pipeline. Change-Id: I13a60b7323ac0bf240f00b3b1360bedd2886e902 --- admin/gitlab-ci/archive.gitlab-ci.yml | 1 - admin/gitlab-ci/documentation.gitlab-ci.yml | 64 +----- admin/gitlab-ci/global.gitlab-ci.yml | 15 +- admin/gitlab-ci/gromacs.gitlab-ci.yml | 218 +------------------- admin/gitlab-ci/lint.gitlab-ci.yml | 6 +- 5 files changed, 21 insertions(+), 283 deletions(-) diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index e1cc6cea43..a41d6c5d90 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -338,7 +338,6 @@ release-verify: webpage-archive-release: extends: - .build-docs-webpage - - .configure-gmxapi-template stage: release-deploy only: refs: diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 2731ef36fe..72f1f37481 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -1,8 +1,5 @@ # Build documentation and web page. .documentation-before-script-template: - variables: - EXTRA_INSTALLS: "" - before_script: - echo $EXTRA_INSTALLS - time apt-get -qq update @@ -17,6 +14,8 @@ # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master stage: configure-build + extends: + - .variables:default only: refs: - web @@ -34,7 +33,6 @@ CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" script: - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then REL_OPTION="-DGMX_BUILD_TARBALL=ON" ; @@ -85,6 +83,8 @@ # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master stage: release-configure + extends: + - .variables:default variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -96,7 +96,6 @@ CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" RELEASE_BUILD_DIR: release-doc-builds RELEASE_SOURCE: release-source-from-tarball RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball @@ -217,20 +216,6 @@ configure-docs: variables: - $GROMACS_RELEASE -configure-docs-gmxapi: - extends: - - .configure-docs - - .configure-gmxapi-template - needs: - - job: simple-build - artifacts: false - variables: - BUILD_DIR: build-docs-gmxapi - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on - except: - variables: - - $GROMACS_RELEASE - .build-docs-binary-template: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script @@ -323,23 +308,6 @@ build-docs: variables: - $GROMACS_RELEASE -build-docs-gmxapi: - extends: - - .build-docs-binary - - .gmxapi-extra-installs - stage: documentation - needs: - - job: simple-build - artifacts: false - - job: configure-docs-gmxapi - artifacts: true - variables: - BUILD_DIR: build-docs-gmxapi - except: - variables: - - $GROMACS_RELEASE - - # The manual build is built separately so that errors in converting # Sphinx to LaTeX and compiling can always be found in the # all-output.txt file, while avoiding many thousands of lines of spam @@ -363,26 +331,9 @@ webpage: variables: - $GROMACS_RELEASE -webpage-gmxapi: - extends: - - .build-docs-webpage - - .gmxapi-extra-installs - stage: post-test - needs: - - job: build-docs-gmxapi - artifacts: true - - job: run-check-source - artifacts: false - variables: - BUILD_DIR: build-docs-gmxapi - except: - variables: - - $GROMACS_RELEASE - webpage-nightly: extends: - .build-docs-webpage - - .configure-gmxapi-template stage: nightly-deploy only: - web @@ -396,10 +347,10 @@ webpage-nightly: artifacts: false - job: linkchecker artifacts: false - - job: webpage-gmxapi + - job: webpage artifacts: true variables: - BUILD_DIR: build-docs-gmxapi + BUILD_DIR: build-docs script: - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ artifacts: @@ -410,7 +361,6 @@ webpage-nightly: webpage-configure-release: extends: - .configure-docs-release - - .configure-gmxapi-template cache: {} only: refs: @@ -430,7 +380,6 @@ webpage-configure-release: webpage-binary-release: extends: - .build-docs-binary - - .gmxapi-extra-installs stage: release-build needs: - job: webpage-configure-release @@ -448,7 +397,6 @@ webpage-binary-release: webpage-build-release: extends: - .build-docs-webpage - - .configure-gmxapi-template dependencies: - webpage-binary-release - webpage-configure-release diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml index c9319c2a32..730a532836 100644 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -12,7 +12,9 @@ CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" BUILD_DIR: build INSTALL_DIR: install - EXTRA_INSTALLS: "" + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON" + EXTRA_INSTALLS: "python3-dev python3-setuptools python3-pip python3-pytest" +# TODO: Get these from updated Docker images. # Behavioral templates @@ -23,17 +25,6 @@ paths: - ccache/ - -.gmxapi-extra-installs: - variables: - EXTRA_INSTALLS: python3-dev python3-networkx python3-setuptools python3-mpi4py python3-numpy python3-pip python3-pytest - -.configure-gmxapi-template: - extends: - - .gmxapi-extra-installs - variables: - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON" - # Tool chains .use-cuda: diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index 90a33670c1..64af06e442 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -108,7 +108,6 @@ simple-build: CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" script: - echo $CMAKE_COMPILER_SCRIPT - echo $CMAKE_EXTRA_OPTIONS @@ -161,7 +160,6 @@ simple-build: CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithAssert" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" RELEASE_BUILD_DIR: release-builds RELEASE_SOURCE: release-source-from-tarball RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball @@ -314,38 +312,6 @@ gromacs:gcc-7-double:release:configure: - package-regressiontests-release - prepare-release-version -gromacs:gcc-7:configure-gmxapi: - extends: - - .gromacs:base:configure - - .configure-gmxapi-template - - .use-gcc7 - needs: - - job: simple-build - artifacts: false - except: - variables: - - $GROMACS_RELEASE - -gromacs:gcc-7:release:configure-gmxapi: - extends: - - .gromacs:base:release:configure - - .configure-gmxapi-template - - .use-gcc7 - cache: {} - variables: - RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - gromacs:gcc-8:configure: extends: - .gromacs:base:configure @@ -381,6 +347,9 @@ gromacs:gcc-8-cuda-10.2:configure: - .gromacs:base:configure - .use-cuda - .use-gcc8:cuda-10.2 + variables: + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" + # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. needs: - job: simple-build artifacts: false @@ -435,38 +404,6 @@ gromacs:clang-7:release:configure: - package-regressiontests-release - prepare-release-version -gromacs:clang-7:configure-gmxapi: - extends: - - .gromacs:base:configure - - .configure-gmxapi-template - - .use-clang7 - needs: - - job: simple-build - artifacts: false - except: - variables: - - $GROMACS_RELEASE - -gromacs:clang-7:release:configure-gmxapi: - extends: - - .gromacs:base:release:configure - - .configure-gmxapi-template - - .use-clang7 - cache: {} - variables: - RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - gromacs:clang-8:configure: extends: - .gromacs:base:configure @@ -590,22 +527,6 @@ gromacs:gcc-7-double:build: variables: - $GROMACS_RELEASE -gromacs:gcc-7:build-gmxapi: - extends: - - .gromacs:base:build - - .use-ccache - - .variables:default - - .gmxapi-extra-installs - - .use-gcc7 - needs: - - job: simple-build - artifacts: false - - job: gromacs:gcc-7:configure-gmxapi - artifacts: true - except: - variables: - - $GROMACS_RELEASE - gromacs:gcc-7-double:release:build: extends: - .gromacs:base:build @@ -626,27 +547,6 @@ gromacs:gcc-7-double:release:build: variables: - $GROMACS_RELEASE -gromacs:gcc-7:release:build-gmxapi: - extends: - - .gromacs:base:build - - .use-ccache - - .variables:default - - .gmxapi-extra-installs - - .use-gcc7 - stage: release-build - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: gromacs:gcc-7:release:configure-gmxapi - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - gromacs:gcc-8:build: extends: - .gromacs:base:build @@ -689,6 +589,9 @@ gromacs:gcc-8-cuda-10.2:build: - .variables:default - .use-cuda - .use-gcc8:cuda-10.2 + variables: + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" + # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. needs: - job: simple-build artifacts: false @@ -748,43 +651,6 @@ gromacs:clang-7:release:build: variables: - $GROMACS_RELEASE -gromacs:clang-7:build-gmxapi: - extends: - - .gromacs:base:build - - .use-ccache - - .variables:default - - .gmxapi-extra-installs - - .use-clang7 - needs: - - job: simple-build - artifacts: false - - job: gromacs:clang-7:configure-gmxapi - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -gromacs:clang-7:release:build-gmxapi: - extends: - - .gromacs:base:build - - .use-ccache - - .variables:default - - .gmxapi-extra-installs - - .use-clang7 - stage: release-build - variables: - BUILD_DIR: release-builds-clang - needs: - - job: gromacs:clang-7:release:configure-gmxapi - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - gromacs:clang-8:build: extends: - .gromacs:base:build @@ -948,41 +814,6 @@ gromacs:gcc-7-double:release:test: variables: - $GROMACS_RELEASE -gromacs:gcc-7:test-gmxapi: - extends: - - .gromacs:base:test - - .gmxapi-extra-installs - - .use-gcc7 - needs: - - job: simple-build - artifacts: false - - job: gromacs:gcc-7:build-gmxapi - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -gromacs:gcc-7:release:test-gmxapi: - extends: - - .gromacs:base:test - - .gmxapi-extra-installs - - .use-gcc7 - stage: release-tests - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: gromacs:gcc-7:release:configure-gmxapi - artifacts: true - - job: gromacs:gcc-7:release:build-gmxapi - artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - gromacs:gcc-8:test: extends: - .gromacs:base:test @@ -1023,6 +854,8 @@ gromacs:gcc-8-cuda-10.2:test: REGRESSIONTEST_PME_RANK_NUMBER: 0 REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 REGRESSIONTEST_OMP_RANK_NUMBER: 1 + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" + # TODO: Enable GMX_PYTHON_PACKAGE when dependencies are in Docker image. extends: - .gromacs:base:test - .use-cuda @@ -1084,41 +917,6 @@ gromacs:clang-7:release:test: variables: - $GROMACS_RELEASE -gromacs:clang-7:test-gmxapi: - extends: - - .gromacs:base:test - - .gmxapi-extra-installs - - .use-clang7 - needs: - - job: simple-build - artifacts: false - - job: gromacs:clang-7:build-gmxapi - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -gromacs:clang-7:release:test-gmxapi: - extends: - - .gromacs:base:test - - .gmxapi-extra-installs - - .use-clang7 - stage: release-tests - variables: - BUILD_DIR: release-builds-clang - needs: - - job: gromacs:clang-7:release:configure-gmxapi - artifacts: true - - job: gromacs:clang-7:release:build-gmxapi - artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - gromacs:clang-8:test: extends: - .gromacs:base:test diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index 361d97c9f8..9ac85814bf 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -11,6 +11,8 @@ configure-clang-tidy: BUILD_DIR: build-clang-tidy CMAKE_EXTRA_OPTIONS: -DCLANG_TIDY=clang-tidy-$COMPILER_MAJOR_VERSION -DGMX_CLANG_TIDY=ON -DGMX_COMPILER_WARNINGS=ON EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" + # TODO: Enable GMX_PYTHON_PACKAGE for clang-tidy when dependencies are in Docker image. except: variables: - $GROMACS_RELEASE @@ -136,7 +138,7 @@ linkchecker: - .build-docs-webpage stage: nightly-build dependencies: - - webpage-gmxapi + - webpage only: - web - schedules @@ -149,7 +151,7 @@ linkchecker: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_LIMIT: 2Gi - BUILD_DIR: build-docs-gmxapi + BUILD_DIR: build-docs script: - cd $BUILD_DIR - linkchecker docs/html/index.html -f ../docs/linkcheckerrc -Fxml --ignore-url html-full -- 2.22.0