From 449c5a1790e8c5df1fb86238cbe15f24a203dcca Mon Sep 17 00:00:00 2001 From: Paul Bauer Date: Wed, 22 Apr 2020 06:52:14 +0000 Subject: [PATCH] Copy of CI from master to 2020 Adjusted branches and build images. Change-Id: I28d011a003119870c41e434cef623765ddc6ace1 --- .gitlab-ci.yml | 1521 +---------------- .../build-and-test-py-gmxapi-0.1.sh | 65 + .../build-and-test-py-gmxapi-0.2.sh | 69 + .../build-and-test-sample_restraint-2020.sh | 57 + .../build-and-test-sample_restraint-2021.sh | 83 + admin/ci-templates/.build-cache-template.yml | 5 - admin/ci-templates/.build-gmxapi-template.yml | 11 - admin/ci-templates/.build-mpi-template.yml | 9 - admin/ci-templates/.build-template.yml | 310 ---- admin/ci-templates/.clang-before-script.yml | 14 - admin/ci-templates/.clang-build-template.yml | 6 - admin/ci-templates/.clang-test-template.yml | 4 - admin/ci-templates/.cuda-template.yml | 8 - .../.documentation-before-script.yml | 13 - admin/ci-templates/.gcc-before-script.yml | 14 - admin/ci-templates/.gcc-build-template.yml | 5 - admin/ci-templates/.gcc-test-template.yml | 4 - admin/ci-templates/.image-templates.yml | 48 - admin/ci-templates/.include-template.yml | 22 - admin/ci-templates/.pull-cache-template.yml | 8 - .../ci-templates/.regressiontest-template.yml | 81 - admin/ci-templates/.release-template.yml | 61 - admin/ci-templates/.tags-template.yml | 3 - admin/ci-templates/.test-runner-template.yml | 13 - admin/ci-templates/.test-script-template.yml | 34 - admin/ci-templates/.variables-template.yml | 11 - admin/clang-tidy.sh | 65 +- admin/containers/buildall.sh | 64 + .../containers/scripted_gmx_docker_builds.py | 501 ++++-- admin/containers/utility.py | 6 +- admin/gitlab-ci/archive.gitlab-ci.yml | 280 +++ .../documentation.gitlab-ci.yml} | 316 ++-- admin/gitlab-ci/global.gitlab-ci.yml | 129 ++ admin/gitlab-ci/gromacs.gitlab-ci.yml | 813 +++++++++ admin/gitlab-ci/lint.gitlab-ci.yml | 173 ++ admin/gitlab-ci/python-gmxapi.gitlab-ci.yml | 108 ++ .../sample_restraint-regression.gitlab-ci.yml | 53 + .../gitlab-ci/sample_restraint.gitlab-ci.yml | 48 + src/CMakeLists.txt | 2 +- src/gromacs/CMakeLists.txt | 10 +- src/gromacs/fileio/matio.cpp | 1 + src/gromacs/selection/CMakeLists.txt | 4 +- src/testutils/TestMacros.cmake | 15 +- 43 files changed, 2600 insertions(+), 2467 deletions(-) create mode 100644 admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh create mode 100644 admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh create mode 100644 admin/ci-scripts/build-and-test-sample_restraint-2020.sh create mode 100644 admin/ci-scripts/build-and-test-sample_restraint-2021.sh delete mode 100644 admin/ci-templates/.build-cache-template.yml delete mode 100644 admin/ci-templates/.build-gmxapi-template.yml delete mode 100644 admin/ci-templates/.build-mpi-template.yml delete mode 100644 admin/ci-templates/.build-template.yml delete mode 100644 admin/ci-templates/.clang-before-script.yml delete mode 100644 admin/ci-templates/.clang-build-template.yml delete mode 100644 admin/ci-templates/.clang-test-template.yml delete mode 100644 admin/ci-templates/.cuda-template.yml delete mode 100644 admin/ci-templates/.documentation-before-script.yml delete mode 100644 admin/ci-templates/.gcc-before-script.yml delete mode 100644 admin/ci-templates/.gcc-build-template.yml delete mode 100644 admin/ci-templates/.gcc-test-template.yml delete mode 100644 admin/ci-templates/.image-templates.yml delete mode 100644 admin/ci-templates/.include-template.yml delete mode 100644 admin/ci-templates/.pull-cache-template.yml delete mode 100644 admin/ci-templates/.regressiontest-template.yml delete mode 100644 admin/ci-templates/.release-template.yml delete mode 100644 admin/ci-templates/.tags-template.yml delete mode 100644 admin/ci-templates/.test-runner-template.yml delete mode 100644 admin/ci-templates/.test-script-template.yml delete mode 100644 admin/ci-templates/.variables-template.yml create mode 100644 admin/containers/buildall.sh create mode 100644 admin/gitlab-ci/archive.gitlab-ci.yml rename admin/{ci-templates/.build-docs-template.yml => gitlab-ci/documentation.gitlab-ci.yml} (55%) create mode 100644 admin/gitlab-ci/global.gitlab-ci.yml create mode 100644 admin/gitlab-ci/gromacs.gitlab-ci.yml create mode 100644 admin/gitlab-ci/lint.gitlab-ci.yml create mode 100644 admin/gitlab-ci/python-gmxapi.gitlab-ci.yml create mode 100644 admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml create mode 100644 admin/gitlab-ci/sample_restraint.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7cd2043d22..60dd3c5c6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,42 +1,5 @@ -include: - - local: '/admin/ci-templates/.include-template.yml' - -.configure-extends-template: - extends: - - .configure-build-template - - .no-cache-template - - .variables-template - - .tags-template - -.configure-release-extends-template: - extends: - - .configure-build-release-template - - .no-cache-template - - .variables-template - - .tags-template - -.build-extends-template: - extends: - - .binary-build-template - - .build-cache-template - - .variables-template - - .tags-template - -.test-extends-template: - extends: - - .test-base-template - - .pull-cache-template - - .test-script-template - - .variables-template - - .tags-template - -.regressiontests-prepare-extends-template: - extends: - - .no-cache-template - - .variables-template - - .tags-template - -# CI is organized into stages so that if early stages fail +# Declare and sequence the stages to which jobs may be assigned. +# The pipeline is organized into stages so that if early stages fail # the later ones are not run. stages: # Initial test build for checking simple compilation and getting regressiontest repo @@ -74,1462 +37,26 @@ stages: # Deploy stage for release builds - release-deploy -# Templates for configuration stage -.configure-gcc: - extends: - - .configure-extends-template - - .gcc-before-script-template - - .build-gcc-template - -.configure-gcc-release: - extends: - - .configure-release-extends-template - - .gcc-before-script-template - - .build-gcc-template - -.configure-gcc-cuda: - extends: - - .configure-gcc - - .cuda-template - -.configure-clang: - extends: - - .configure-extends-template - - .clang-before-script-template - - .build-clang-template - -.configure-clang-release: - extends: - - .configure-release-extends-template - - .clang-before-script-template - - .build-clang-template - -.configure-docs: - extends: - - .configure-extends-template - - .configure-docs-template - - .documentation-before-script-template - -.configure-docs-release: - extends: - - .configure-extends-template - - .configure-docs-release-template - - .documentation-before-script-template - -# Templates for build stage -.build-archive: - extends: - - .build-extends-template - - .documentation-before-script-template - - .build-docs-binary-template - - .archive-build-template - -.build-gcc: - extends: - - .build-extends-template - - .gcc-before-script-template - - .build-gcc-template - -.build-clang: - extends: - - .build-extends-template - - .clang-before-script-template - - .build-clang-template - -.build-docs-binary: - extends: - - .build-extends-template - - .build-cache-template - - .build-docs-binary-template - - .documentation-before-script-template - -.build-docs-webpage: - extends: - - .binary-build-template - - .variables-template - - .tags-template - - .no-cache-template - - .build-docs-webpage-template - - .documentation-before-script-template - -.build-gcc-cuda: - extends: - - .build-gcc - - .cuda-template - -# Templates for test stage -.test-gcc: - extends: - - .test-extends-template - - .gcc-before-script-template - - .test-gcc-template - -.test-gcc-cuda: - extends: - - .test-gcc - - .cuda-template - -.test-clang: - extends: - - .test-extends-template - - .clang-before-script-template - - .test-clang-template - -# Special job to package regressiontest files and have them available for testing -# Runs during pre-build -prepare-regressiontests: - extends: - - .regressiontests-prepare-extends-template - - .regressiontests-template - -prepare-release-version: - extends: - - .regressiontests-prepare-extends-template - - .release-version-template - -# Initial build that always needs to pass before follow-up stages run -simple-build: - extends: - - .simple-build-template - - .build-cache-template - - .variables-template - - .tags-template - - .gcc-before-script-template - - .build-gcc-template - - .gcc7-template - variables: - COMPILER_MAJOR_VERSION: 7 - BUILD_DIR: simple-build - except: - variables: - - $GROMACS_RELEASE - -# Jobs running in first pre-build stage -configure-gcc-7: - extends: - - .configure-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - except: - variables: - - $GROMACS_RELEASE - -configure-gcc-7-gmxapi: - extends: - - .configure-gcc - - .configure-gmxapi-template - - .gcc7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - except: - variables: - - $GROMACS_RELEASE - -configure-gcc-7-double: - extends: - - .configure-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - CMAKE_PRECISION_OPTIONS: -DGMX_DOUBLE=ON - except: - variables: - - $GROMACS_RELEASE - -configure-gcc-8: - extends: - - .configure-gcc - - .gcc8-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 8 - except: - variables: - - $GROMACS_RELEASE - -configure-gcc-8-cuda-10.2: - extends: - - .configure-gcc-cuda - - .gcc-8-cuda-10.2-template - needs: - - job: simple-build - artifacts: false - except: - variables: - - $GROMACS_RELEASE - -configure-gcc-7-simd-sse41: - extends: - - .configure-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" - except: - variables: - - $GROMACS_RELEASE - -configure-clang-7: - extends: - - .configure-clang - - .clang7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - except: - variables: - - $GROMACS_RELEASE - -configure-clang-7-gmxapi: - extends: - - .configure-clang - - .configure-gmxapi-template - - .clang7-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 7 - except: - variables: - - $GROMACS_RELEASE - -configure-clang-8: - extends: - - .configure-clang - - .clang8-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: 8 - except: - variables: - - $GROMACS_RELEASE - -# For version of clang before 7, we need to always provide the full version -# Use something like this instead in the download script: -# APT_COMPILER_VERSION=$(($COMPILER_MAJOR_VERSION < 7 ? "${COMPILER_MAJOR_VERSION}.0" : $COMPILER_MAJOR_VERSION)) -configure-clang-6: - extends: - - .configure-clang - - .clang6-template - needs: - - job: simple-build - artifacts: false - variables: - COMPILER_MAJOR_VERSION: "6.0" - CMAKE_MPI_OPTIONS: -DGMX_OPENMP=OFF - except: - variables: - - $GROMACS_RELEASE - -configure-clang-tidy: - extends: - - .configure-clang - - .clang7-template - needs: - - job: simple-build - artifacts: false - variables: - BUILD_DIR: build-clang-tidy - COMPILER_MAJOR_VERSION: 7 - CMAKE_EXTRA_OPTIONS: -DCLANG_TIDY=clang-tidy-$COMPILER_MAJOR_VERSION -DGMX_CLANG_TIDY=ON -DGMX_COMPILER_WARNINGS=ON - EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION - except: - variables: - - $GROMACS_RELEASE - -configure-docs: - extends: - - .configure-docs - needs: - - job: simple-build - artifacts: false - variables: - BUILD_DIR: build-docs - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on - except: - variables: - - $GROMACS_RELEASE - -configure-docs-gmxapi: - extends: - - .configure-docs - - .configure-gmxapi-template - needs: - - job: simple-build - artifacts: false - variables: - BUILD_DIR: build-docs-gmxapi - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on - except: - variables: - - $GROMACS_RELEASE - -# Only running in scheduled pipelines that are not release builds -configure-archive-nightly: - stage: nightly-configure - only: - - web - - triggers - - schedules - needs: - - job: simple-build - artifacts: false - except: - variables: - - $GROMACS_RELEASE - extends: - - .configure-docs - variables: - BUILD_DIR: build-package - CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF - CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF - -configure-archive-release: - only: - refs: - - web - - triggers - - schedules - variables: - - $GROMACS_RELEASE - - extends: - - .configure-docs - variables: - BUILD_DIR: build-package - CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF - CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF - -# Jobs running during build stage -build-gcc-7: - extends: - - .build-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-7 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - - -build-gcc-7-gmxapi: - extends: - - .build-gcc - - .gmxapi-extra-installs - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-7-gmxapi - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-gcc-7-double: - extends: - - .build-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-7-double - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-gcc-8: - extends: - - .build-gcc - - .gcc8-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-8 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-gcc-8-cuda-10.2: - extends: - - .build-gcc-cuda - - .gcc-8-cuda-10.2-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-8-cuda-10.2 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-gcc-7-simd-sse41: - extends: - - .build-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: configure-gcc-7-simd-sse41 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-clang-7: - extends: - - .build-clang - - .clang7-template - needs: - - job: simple-build - artifacts: false - - job: configure-clang-7 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-clang-7-gmxapi: - extends: - - .build-clang - - .gmxapi-extra-installs - - .clang7-template - needs: - - job: simple-build - artifacts: false - - job: configure-clang-7-gmxapi - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-clang-8: - extends: - - .build-clang - - .clang8-template - needs: - - job: simple-build - artifacts: false - - job: configure-clang-8 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -build-clang-6: - extends: - - .build-clang - - .clang6-template - needs: - - job: simple-build - artifacts: false - - job: configure-clang-6 - artifacts: true - except: - variables: - - $GROMACS_RELEASE - -# Jobs running during test stage -test-gcc-7: - extends: - - .test-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: build-gcc-7 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-gcc-7-gmxapi: - extends: - - .test-gcc - - .gmxapi-extra-installs - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: build-gcc-7-gmxapi - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-gcc-7-double: - extends: - - .test-gcc - - .gcc7-template - variables: - REGRESSIONTEST_DOUBLE: "-double" - needs: - - job: simple-build - artifacts: false - - job: build-gcc-7-double - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-gcc-8: - extends: - - .test-gcc - - .gcc8-template - needs: - - job: simple-build - artifacts: false - - job: build-gcc-8 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-gcc-8-cuda-10.2: - variables: - KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu" - KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 - extends: - - .test-gcc-cuda - - .gcc-8-cuda-10.2-template - variables: - REGRESSIONTEST_PME_RANK_NUMBER: 0 - REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 - REGRESSIONTEST_OMP_RANK_NUMBER: 1 - needs: - - job: simple-build - artifacts: false - - job: build-gcc-8-cuda-10.2 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-gcc-7-simd-sse41: - extends: - - .test-gcc - - .gcc7-template - needs: - - job: simple-build - artifacts: false - - job: build-gcc-7-simd-sse41 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-clang-7: - extends: - - .test-clang - - .clang7-template - needs: - - job: simple-build - artifacts: false - - job: build-clang-7 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-clang-7-gmxapi: - extends: - - .test-clang - - .gmxapi-extra-installs - - .clang7-template - needs: - - job: simple-build - artifacts: false - - job: build-clang-7-gmxapi - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-clang-8: - extends: - - .test-clang - - .clang8-template - needs: - - job: simple-build - artifacts: false - - job: build-clang-8 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -test-clang-6: - extends: - - .test-clang - - .clang6-template - variables: - REGRESSIONTEST_OMP_RANK_NUMBER: 0 - needs: - - job: simple-build - artifacts: false - - job: build-clang-6 - - job: prepare-regressiontests - except: - variables: - - $GROMACS_RELEASE - -# Jobs running in source checking stage -build-clang-tidy: - extends: - - .build-clang - - .clang7-template - stage: source-check - needs: - - job: simple-build - artifacts: false - - job: configure-clang-tidy - artifacts: true - variables: - BUILD_DIR: build-clang-tidy - COMPILER_MAJOR_VERSION: 7 - EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION - except: - variables: - - $GROMACS_RELEASE - -run-clang-format: - extends: - - .binary-build-template - - .no-cache-template - - .variables-template - - .tags-template - - .clang-before-script-template - - .build-clang-template - - .clang7-template - stage: source-check - when: always - needs: - - job: simple-build - artifacts: false - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - COMPILER_MAJOR_VERSION: 7 - EXTRA_INSTALLS: clang-format-$COMPILER_MAJOR_VERSION - except: - variables: - - $GROMACS_RELEASE - script: - - export CLANG_FORMAT=clang-format-$COMPILER_MAJOR_VERSION - - admin/clang-format.sh check --rev=HEAD^ --warnings=clang-format.log - - grep -iq "needs formatting" clang-format.log | tee formatting.txt || true - - if [ -s formatting.txt ] ; then echo "clang-format.sh found issues"; exit 1; fi - artifacts: - when: on_failure - paths: - - clang-format.log - - formatting.txt - -run-copyright-check: - extends: - - .binary-build-template - - .no-cache-template - - .variables-template - - .tags-template - - .clang-before-script-template - - .build-clang-template - - .clang7-template - - stage: source-check - when: always - needs: - - job: simple-build - artifacts: false - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - except: - variables: - - $GROMACS_RELEASE - script: - - admin/copyright.sh check --rev=HEAD^ --warnings=copyright.log - - grep -iq "copyright year" copyright.log | tee years.log || true - - grep -iq "copyright header" copyright.log | tee headers.log || true - - if [[ -s years.log || -s headers.log ]] ; then - echo "Copyright information needs updating" ; - exit 1 ; - fi - artifacts: - when: on_failure - paths: - - copyright.log - - years.log - - headers.log - -run-check-source: - extends: - - .no-cache-template - - .variables-template - - .tags-template - - .build-docs-binary-template - - .documentation-before-script-template - stage: source-check - needs: - - job: simple-build - artifacts: false - - job: build-docs - artifacts: true - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - BUILD_DIR: build-docs - COMPILER_MAJOR_VERSION: 7 - except: - variables: - - $GROMACS_RELEASE - - script: - - cd $BUILD_DIR - - cmake --build . --target check-source - - grep -qi "error" docs/doxygen/doxygen*log docs/doxygen/check-source.log | tee doxygenError.txt || true - - grep -qi "warning" docs/doxygen/doxygen*log docs/doxygen/check-source.log | tee -a doxygenError.txt || true - - if [ -s doxygenError.txt ] ; then echo "Found errors while running doxygen"; exit 1; fi - artifacts: - name: docs-artifacts-$CI_COMMIT_REF_SLUG - when: always - expire_in: 1 week - paths: - - $BUILD_DIR/docs/doxygen/doxygen-xml.log - - $BUILD_DIR/docs/doxygen/check-source.log - -# Jobs running during documentation build stage -build-docs: - extends: - - .build-docs-binary - stage: documentation - needs: - - job: simple-build - artifacts: false - - job: configure-docs - artifacts: true - variables: - BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE - -build-docs-gmxapi: - extends: - - .build-docs-binary - - .gmxapi-extra-installs - stage: documentation - needs: - - job: simple-build - artifacts: false - - job: configure-docs-gmxapi - artifacts: true - variables: - BUILD_DIR: build-docs-gmxapi - except: - variables: - - $GROMACS_RELEASE - -# Jobs running in post-test stage - -# The manual build is built separately so that errors in converting -# Sphinx to LaTeX and compiling can always be found in the -# all-output.txt file, while avoiding many thousands of lines of spam -# from pdflatex for normal builds. This does reduce the available -# parallel utilization, and so increases the build time. -# -# TODO why are the doxygen and sphinx log files disappearing -# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job -webpage: - extends: - - .build-docs-webpage - stage: post-test - needs: - - job: build-docs - artifacts: true - - job: run-check-source - artifacts: false - variables: - BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE - -webpage-gmxapi: - extends: - - .build-docs-webpage - - .gmxapi-extra-installs - stage: post-test - needs: - - job: build-docs-gmxapi - artifacts: true - - job: run-check-source - artifacts: false - variables: - BUILD_DIR: build-docs-gmxapi - except: - variables: - - $GROMACS_RELEASE - - -# Jobs running in release stage -linkchecker: - extends: - - .build-docs-webpage - stage: nightly-build - dependencies: - - webpage-gmxapi - only: - - web - - schedules - - triggers - - merge_requests - except: - variables: - - $GROMACS_RELEASE - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - BUILD_DIR: build-docs-gmxapi - script: - - cd $BUILD_DIR - - linkchecker docs/html/index.html -f ../docs/linkcheckerrc -Fxml --ignore-url html-full - --ignore-url html-user --ignore-url html-lib --ignore-url .tar.gz --ignore-url _sources - -o xml - artifacts: - reports: - junit: $BUILD_DIR/linkchecker-out.xml - -build-archive-nightly: - extends: - - .build-archive - stage: nightly-build - needs: - - job: simple-build - artifacts: false - - job: configure-archive-nightly - artifacts: true - only: - - web - - triggers - - schedules - except: - variables: - - $GROMACS_RELEASE - variables: - BUILD_DIR: build-package - -webpage-nightly: - extends: - - .build-docs-webpage - - .configure-gmxapi-template - stage: nightly-deploy - only: - - web - - triggers - - schedules - except: - variables: - - $GROMACS_RELEASE - needs: - - job: simple-build - artifacts: false - - job: linkchecker - artifacts: false - - job: webpage-gmxapi - artifacts: true - variables: - BUILD_DIR: build-docs-gmxapi - script: - - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ - artifacts: - when: always - paths: - - webpage.tar.gz - -package-regressiontests-release: - extends: - - .regressiontests-prepare-extends-template - - .regressiontests-release-template - needs: - - job: prepare-release-version - artifacts: true - - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - - -package-archive-release: - extends: - - .build-archive - stage: release-package - needs: - - job: configure-archive-release - artifacts: true - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - variables: - BUILD_DIR: build-package - -release-verify: - extends: - - .regressiontests-prepare-extends-template - - .release-verify-template - - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -webpage-configure-release: - extends: - - .configure-docs-release - - .configure-gmxapi-template - - .no-cache-template - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - variables: - CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on - -configure-gcc-7-release: - extends: - - .configure-gcc-release - - .no-cache-template - - .gcc7-template - variables: - COMPILER_MAJOR_VERSION: 7 - RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-gcc-7-gmxapi-release: - extends: - - .configure-gcc-release - - .configure-gmxapi-template - - .no-cache-template - - .gcc7-template - variables: - COMPILER_MAJOR_VERSION: 7 - RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-gcc-7-double-release: - extends: - - .configure-gcc-release - - .no-cache-template - - .gcc7-template - variables: - COMPILER_MAJOR_VERSION: 7 - CMAKE_PRECISION_OPTIONS: -DGMX_DOUBLE=ON - RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-gcc-8-release: - extends: - - .configure-gcc-release - - .no-cache-template - - .gcc8-template - variables: - COMPILER_MAJOR_VERSION: 8 - RELEASE_BUILD_DIR: release-builds-gcc - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-clang-7-release: - extends: - - .configure-clang-release - - .no-cache-template - - .clang7-template - variables: - COMPILER_MAJOR_VERSION: 7 - RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-clang-7-gmxapi-release: - extends: - - .configure-clang-release - - .configure-gmxapi-template - - .no-cache-template - - .clang7-template - variables: - COMPILER_MAJOR_VERSION: 7 - RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -configure-clang-8-release: - extends: - - .configure-clang-release - - .no-cache-template - - .clang8-template - variables: - COMPILER_MAJOR_VERSION: 8 - RELEASE_BUILD_DIR: release-builds-clang - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - dependencies: - - package-archive-release - - package-regressiontests-release - - prepare-release-version - -build-gcc-7-release: - extends: - - .build-gcc - - .gcc7-template - stage: release-build - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-7-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-gcc-7-gmxapi-release: - extends: - - .build-gcc - - .gmxapi-extra-installs - - .gcc7-template - stage: release-build - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-7-gmxapi-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-gcc-7-double-release: - extends: - - .build-gcc - - .gcc7-template - stage: release-build - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-7-double-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-gcc-8-release: - extends: - - .build-gcc - - .gcc8-template - stage: release-build - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-8-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-clang-7-release: - extends: - - .build-clang - - .clang7-template - stage: release-build - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-7-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-clang-7-gmxapi-release: - extends: - - .build-clang - - .gmxapi-extra-installs - - .clang7-template - stage: release-build - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-7-gmxapi-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -build-clang-8-release: - extends: - - .build-clang - - .clang8-template - stage: release-build - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-8-release - artifacts: true - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - -webpage-binary-release: - extends: - - .build-docs-binary - - .gmxapi-extra-installs - stage: release-build - needs: - - job: webpage-configure-release - artifacts: true - variables: - BUILD_DIR: release-doc-builds - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-gcc-7-release: - extends: - - .test-gcc - - .gcc7-template - stage: release-tests - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-7-release - artifacts: true - - job: build-gcc-7-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-gcc-7-gmxapi-release: - extends: - - .test-gcc - - .gmxapi-extra-installs - - .gcc7-template - stage: release-tests - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-7-gmxapi-release - artifacts: true - - job: build-gcc-7-gmxapi-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-gcc-7-double-release: - extends: - - .test-gcc - - .gcc7-template - stage: release-tests - variables: - BUILD_DIR: release-builds-gcc - REGRESSIONTEST_DOUBLE: "-double" - needs: - - job: configure-gcc-7-double-release - artifacts: true - - job: build-gcc-7-double-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-gcc-8-release: - extends: - - .test-gcc - - .gcc8-template - stage: release-tests - variables: - BUILD_DIR: release-builds-gcc - needs: - - job: configure-gcc-8-release - artifacts: true - - job: build-gcc-8-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-clang-7-release: - extends: - - .test-clang - - .clang7-template - stage: release-tests - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-7-release - artifacts: true - - job: build-clang-7-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-clang-7-gmxapi-release: - extends: - - .test-clang - - .gmxapi-extra-installs - - .clang7-template - stage: release-tests - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-7-gmxapi-release - artifacts: true - - job: build-clang-7-gmxapi-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -test-clang-8-release: - extends: - - .test-clang - - .clang8-template - stage: release-tests - variables: - BUILD_DIR: release-builds-clang - needs: - - job: configure-clang-8-release - artifacts: true - - job: build-clang-8-release - artifacts: true - - job: prepare-regressiontests - only: - refs: - - triggers - - web - - schedules - variables: - - $GROMACS_RELEASE - -webpage-build-release: - extends: - - .build-docs-webpage - - .configure-gmxapi-template - - dependencies: - - webpage-binary-release - - webpage-configure-release - - stage: release-prepare-deploy - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - variables: - BUILD_DIR: release-doc-builds - -webpage-archive-release: - extends: - - .build-docs-webpage - - .configure-gmxapi-template - stage: release-deploy - only: - refs: - - web - - triggers - - schedules - variables: - - $GROMACS_RELEASE - dependencies: - - webpage-build-release - - package-archive-release - - package-regressiontests-release - variables: - BUILD_DIR: release-doc-builds - script: - - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ - artifacts: - when: always - paths: - - webpage.tar.gz - - gromacs-*tar.gz - - regressiontests-*tar.gz +default: + tags: + - k8s-scilifelab +include: + # Mix-in helper job definitions of general utility. + - local: '/admin/gitlab-ci/global.gitlab-ci.yml' + # gmxapi Python package. + - local: '/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml' + # Further API validation and usability of sample gmxapi extension package. + - local: '/admin/gitlab-ci/sample_restraint.gitlab-ci.yml' + # API regression testing using sample gmxapi extension package. + - local: '/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml' + # Build and test GROMACS in a variety of environments and configurations. + # To do: Consider expanding matrix here to improve transparency and reduce file sizes. + # E.g. '/admin/gitlab-ci/matrix/clang-8.gitlab-ci.yml + - local: '/admin/gitlab-ci/gromacs.gitlab-ci.yml' + # Repository cleanliness. Source tidiness, linting, and policy compliance. + - local: '/admin/gitlab-ci/lint.gitlab-ci.yml' + # Web page and manual. + - local: '/admin/gitlab-ci/documentation.gitlab-ci.yml' + # Prepare release archives. + - local: '/admin/gitlab-ci/archive.gitlab-ci.yml' diff --git a/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh b/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh new file mode 100644 index 0000000000..9c405de8d7 --- /dev/null +++ b/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Build, install, and test the gmxapi 0.1.x Python package distributed with +# GROMACS 2020. +# +# It is assumed that this script is run in an activated Python venv with the +# gmxapi dependencies already installed, with `python` resolvable by the shell +# to the appropriate Python interpreter. +# +# This script is intended to support automated GROMACS testing infrastructure, +# and may be removed without notice. +# +# WARNING: This script assumes OpenMPI mpiexec. Syntax for launch wrappers from +# other implementations will need different syntax, and we should get a +# MPIRUNNER from the environment, or something. + +# Make sure the script errors if any commands error. +set -ev + +# Create "sdist" source distribution archive. +pushd python_packaging/src + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python setup.py sdist + # TODO: Identify SDIST + + # Build and install from sdist. + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python -m pip install \ + --no-cache-dir \ + --no-deps \ + --no-index \ + --no-build-isolation \ + dist/gmxapi* + # TODO: Build and install from $SDIST instead of wildcard. +popd + +# Run Python unit tests. +python -m pytest $PWD/python_packaging/src/test --junitxml=$PY_UNIT_TEST_XML +# TODO: enable MPI tests +#if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/python_packaging/src/test \ +# --junitxml=$PY_MPI_UNIT_TEST_XML +#fi + +# Run Python acceptance tests. +python -m pytest $PWD/python_packaging/test --junitxml=$PY_ACCEPTANCE_TEST_XML +# TODO: enable MPI tests +#if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/python_packaging/test \ +# --junitxml=$PY_MPI_ACCEPTANCE_TEST_XML +#fi diff --git a/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh b/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh new file mode 100644 index 0000000000..2bd12b8805 --- /dev/null +++ b/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Build, install, and test the gmxapi 0.2 Python package developed with +# GROMACS 2021. +# +# This script assumes an activated Python venv with the +# gmxapi dependencies already installed, with `python` resolvable by the shell +# to the appropriate Python interpreter. +# +# This script is intended to support automated GROMACS testing infrastructure, +# and may be removed without notice. +# +# WARNING: This script assumes OpenMPI mpiexec. Syntax for launch wrappers from +# other implementations will need different syntax, and we should get a +# MPIRUNNER from the environment, or something. + +# Make sure the script errors if any commands error. +set -ev + +# Create "sdist" source distribution archive. +pushd python_packaging/src + # TODO: Remove extraneous environment variable with resolution of #3273 + # Ref: https://redmine.gromacs.org/issues/3273 + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python setup.py sdist + # TODO: Identify SDIST + + # Build and install from sdist. + # Note that tool chain may be provided differently in GROMACS 2020 and 2021. + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python -m pip install \ + --no-cache-dir \ + --no-deps \ + --no-index \ + --no-build-isolation \ + dist/gmxapi* + # TODO: Build and install from $SDIST instead of wildcard. + +popd + +# Run Python unit tests. +python -m pytest python_packaging/src/test --junitxml=$PY_UNIT_TEST_XML +# TODO: enable MPI tests +#if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/python_packaging/src/test \ +# --junitxml=$PY_MPI_UNIT_TEST_XML +#fi + +# Run Python acceptance tests. +python -m pytest python_packaging/test --junitxml=$PY_ACCEPTANCE_TEST_XML +# TODO: enable MPI tests +#if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/python_packaging/test \ +# --junitxml=$PY_MPI_ACCEPTANCE_TEST_XML +#fi diff --git a/admin/ci-scripts/build-and-test-sample_restraint-2020.sh b/admin/ci-scripts/build-and-test-sample_restraint-2020.sh new file mode 100644 index 0000000000..964660dcf8 --- /dev/null +++ b/admin/ci-scripts/build-and-test-sample_restraint-2020.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Build and test the sample_restraint package distributed with GROMACS 2020. +# +# This script is intended to support automated GROMACS testing infrastructure, +# and may be removed without notice. +# +# WARNING: This script assumes OpenMPI mpiexec. Syntax for launch wrappers from +# other implementations will need different syntax, and we should get a +# MPIRUNNER from the environment, or something. + +# Make sure the script errors if any commands error. +set -ev + +source $VENVPATH/bin/activate + +# TODO: We should be able to just use a $GMXAPI_0_1_SDIST or venv artifact... +pushd python_packaging/src + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python -m pip install \ + --no-cache-dir \ + --no-deps \ + --no-index \ + --no-build-isolation \ + . +popd + +. $INSTALL_DIR/bin/GMXRC +pushd python_packaging/sample_restraint + mkdir build + pushd build + # TODO: Update with respect to https://redmine.gromacs.org/issues/3133 + cmake .. \ + -DDOWNLOAD_GOOGLETEST=ON \ + -DGMXAPI_EXTENSION_DOWNLOAD_PYBIND=ON + make + + make test + #TODO: Can we get ctest JUnitXML output here? + + make install + popd + + python -m pytest $PWD/tests --junitxml=$PLUGIN_TEST_XML +# TODO: enable MPI tests +# if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/tests \ +# --junitxml=$PLUGIN_MPI_TEST_XML +# fi +popd diff --git a/admin/ci-scripts/build-and-test-sample_restraint-2021.sh b/admin/ci-scripts/build-and-test-sample_restraint-2021.sh new file mode 100644 index 0000000000..72aec86e20 --- /dev/null +++ b/admin/ci-scripts/build-and-test-sample_restraint-2021.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Build and test the sample_restraint package distributed with GROMACS 2021. +# +# This script is intended to support automated GROMACS testing infrastructure, +# and may be removed without notice. +# +# WARNING: This script assumes OpenMPI mpiexec. Syntax for launch wrappers from +# other implementations will need different syntax, and we should get a +# MPIRUNNER from the environment, or something. + +# Make sure the script errors if any commands error. +set -ev + +source $VENVPATH/bin/activate + +# Create "sdist" source distribution archive. +pushd python_packaging/src + # TODO: Remove extraneous environment variable with resolution of #3273 + # Ref: https://redmine.gromacs.org/issues/3273 + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python setup.py sdist + # TODO: Identify SDIST + + # Build and install from sdist. + # Note that tool chain may be provided differently in GROMACS 2020 and 2021. + # TODO: How should we distinguish the target branch? +# if [ "2020" -eq "$GROMACS_VERSION" ]; then + if [ 1 ]; then + GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ + python -m pip install \ + --no-cache-dir \ + --no-deps \ + --no-index \ + --no-build-isolation \ + dist/gmxapi* + # TODO: Build and install from $SDIST instead of wildcard. + # TODO: How should we distinguish the target branch? +# elif [ "2021" -eq "$GROMACS_VERSION" ]; then +# GMXTOOLCHAINDIR=$INSTALL_DIR/share/cmake/gromacs \ +# python -m pip install \ +# --no-cache-dir \ +# --no-deps \ +# --no-index \ +# --no-build-isolation \ +# dist/gmxapi* + # TODO: Build and install from $SDIST instead of wildcard. + else + echo "Logic error in GROMACS version handling." + exit 1 + fi +popd + +. $INSTALL_DIR/bin/GMXRC +pushd python_packaging/sample_restraint + mkdir build + pushd build + # TODO: Update with respect to https://redmine.gromacs.org/issues/3133 + cmake .. \ + -DDOWNLOAD_GOOGLETEST=ON \ + -DGMXAPI_EXTENSION_DOWNLOAD_PYBIND=ON + make + + make test + #TODO: Can we get ctest JUnitXML output here? + + make install + popd + + python -m pytest $PWD/tests --junitxml=$PLUGIN_TEST_XML +# TODO: enable MPI tests +# if [ -x `which mpiexec` ]; then +# PYTHONDONTWRITEBYTECODE=1 \ +# mpiexec --allow-run-as-root \ +# --mca opal_warn_on_missing_libcuda 0 \ +# --mca orte_base_help_aggregate 0 \ +# -n 2 \ +# `which python` -m pytest \ +# -p no:cacheprovider \ +# $PWD/tests \ +# --junitxml=$PLUGIN_MPI_TEST_XML +# fi +popd diff --git a/admin/ci-templates/.build-cache-template.yml b/admin/ci-templates/.build-cache-template.yml deleted file mode 100644 index c65c8333e3..0000000000 --- a/admin/ci-templates/.build-cache-template.yml +++ /dev/null @@ -1,5 +0,0 @@ -.build-cache-template: - cache: - key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - ccache/ diff --git a/admin/ci-templates/.build-gmxapi-template.yml b/admin/ci-templates/.build-gmxapi-template.yml deleted file mode 100644 index 178b212cca..0000000000 --- a/admin/ci-templates/.build-gmxapi-template.yml +++ /dev/null @@ -1,11 +0,0 @@ -.gmxapi-extra-installs: - variables: - EXTRA_INSTALLS: python3-dev python3-networkx python3-setuptools python3-mpi4py python3-numpy python3-pip python3-pytest - - -.configure-gmxapi-template: - extends: - - .gmxapi-extra-installs - variables: - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON" - diff --git a/admin/ci-templates/.build-mpi-template.yml b/admin/ci-templates/.build-mpi-template.yml deleted file mode 100644 index 3bf986a0a3..0000000000 --- a/admin/ci-templates/.build-mpi-template.yml +++ /dev/null @@ -1,9 +0,0 @@ -.mpi-extra-installs: - variables: - EXTRA_INSTALLS: libopenmpi-dev - -.build-mpi-template: - extends: - - .mpi-extra-installs - variables: - CMAKE_MPI_OPTIONS: "-DGMX_MPI=ON" diff --git a/admin/ci-templates/.build-template.yml b/admin/ci-templates/.build-template.yml deleted file mode 100644 index 1045801237..0000000000 --- a/admin/ci-templates/.build-template.yml +++ /dev/null @@ -1,310 +0,0 @@ -.configure-build-template: - # Dockerfiles are from dockerhub, user eriklindahl - # image in admin/dockerimages/ci-docs-py27 - stage: configure-build - only: - refs: - - web - - pushes - - schedules - - merge_requests - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - CMAKE_COMPILER_SCRIPT: "" - CMAKE_EXTRA_OPTIONS: "" - CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" - CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" - CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" - CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" - CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" - - script: - - echo $CMAKE_COMPILER_SCRIPT - - echo $CMAKE_EXTRA_OPTIONS - - echo $CMAKE_SIMD_OPTIONS - - echo $CMAKE_GPU_OPTIONS - - echo $CMAKE_MPI_OPTIONS - - echo $CMAKE_PRECISION_OPTIONS - - echo $CMAKE_BUILD_TYPE_OPTIONS - - echo $CMAKE_GMXAPI_OPTIONS - - if [[ -d $BUILD_DIR ]] ; then - rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; - else - echo "Preparing new build directory" ; - mkdir $BUILD_DIR ; - fi - - cd $BUILD_DIR - - cmake .. - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - $CMAKE_COMPILER_SCRIPT - $CMAKE_EXTRA_OPTIONS - $CMAKE_SIMD_OPTIONS - $CMAKE_MPI_OPTIONS - $CMAKE_PRECISION_OPTIONS - $CMAKE_BUILD_TYPE_OPTIONS - $CMAKE_GPU_OPTIONS - $CMAKE_GMXAPI_OPTIONS - -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON - 2>&1 | tee cmakeLog.log - - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - - cd .. - artifacts: - when: always - paths: - - $BUILD_DIR - -.configure-build-release-template: - # Dockerfiles are from dockerhub, user eriklindahl - # image in admin/dockerimages/ci-docs-py27 - stage: release-configure - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - CMAKE_COMPILER_SCRIPT: "" - CMAKE_EXTRA_OPTIONS: "" - CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" - CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" - CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" - CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithAssert" - CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" - RELEASE_BUILD_DIR: release-builds - RELEASE_SOURCE: release-source-from-tarball - RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball - - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - script: - - VERSION=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` - - if [[ $GROMACS_RELEASE != "true" ]] ; then - VERSION=$VERSION-dev ; - fi - - SOURCENAME=gromacs-$VERSION - - SOURCETARBALL=$SOURCENAME.tar.gz - - REGTESTNAME=regressiontests-$VERSION - - REGTESTTARBALL=$REGTESTNAME.tar.gz - - echo "$SOURCETARBALL" - - echo "$REGTESTTARBALL" - - RELEASE_TARBALLS=release-tarballs - - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS - - mv $SOURCETARBALL $RELEASE_TARBALLS - - mv $REGTESTTARBALL $RELEASE_TARBALLS - - cd $RELEASE_TARBALLS -# We rename the source and regressiontest directories -# to have names for them that don't change for different versions. - - tar -xf $SOURCETARBALL - - mv $SOURCENAME ../$RELEASE_SOURCE - - tar -xf $REGTESTTARBALL - - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS - - rm $SOURCETARBALL - - cp $REGTESTTARBALL gmx-regressiontests.tar.gz - - cd .. - - echo $CMAKE_COMPILER_SCRIPT - - echo $CMAKE_EXTRA_OPTIONS - - echo $CMAKE_SIMD_OPTIONS - - echo $CMAKE_GPU_OPTIONS - - echo $CMAKE_MPI_OPTIONS - - echo $CMAKE_PRECISION_OPTIONS - - echo $CMAKE_BUILD_TYPE_OPTIONS - - echo $CMAKE_GMXAPI_OPTIONS - - if [[ -d $RELEASE_BUILD_DIR ]] ; then - echo "Cleaning up build directory" ; - rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ; - else - echo "Preparing new build directory" ; - mkdir $RELEASE_BUILD_DIR ; - fi - - cd $RELEASE_BUILD_DIR - - cmake ../$RELEASE_SOURCE/ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - $CMAKE_COMPILER_SCRIPT - $CMAKE_EXTRA_OPTIONS - $CMAKE_SIMD_OPTIONS - $CMAKE_MPI_OPTIONS - $CMAKE_PRECISION_OPTIONS - $CMAKE_BUILD_TYPE_OPTIONS - $CMAKE_GPU_OPTIONS - $CMAKE_GMXAPI_OPTIONS - "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS" - -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON - 2>&1 | tee cmakeLog.log - - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - - cd .. - artifacts: - when: always - paths: - - $RELEASE_BUILD_DIR - - $RELEASE_REGRESSIONTESTS - - $RELEASE_SOURCE - -.binary-build-template: - # Dockerfiles are from dockerhub, user eriklindahl - # image in admin/dockerimages/ci-docs-py27 - stage: build - only: - refs: - - web - - pushes - - schedules - - merge_requests - script: - - cd $BUILD_DIR - - cmake --build . -- -j8 2>&1 | tee buildLogFile.log - - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log - - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log - | grep -v "CMake" | tee buildErrors.log || true - - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true - - cmake --build . --target install 2>&1 | tee installBuildLogFile.log - - if [ -s buildErrors.log ] ; then echo "Found compiler warning during build"; cat buildErrors.log; exit 1; fi - - cd .. - artifacts: - when: always - paths: - - $BUILD_DIR - - $INSTALL_DIR/ - -# Right now we need to use an ugly hack to move the original build directory -# and the ccache files out of the build archive, as it is not possible to use -# the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203 -# As soon as this is resolved we should use the separate build directories instead of -# manually hacking things. TODO -# Redmine #3361 -.archive-build-template: - # Dockerfiles are from dockerhub, user eriklindahl - image: gromacs/ci-docs-clang:2020 - only: - refs: - - web - - pushes - - schedules - - merge_requests - variables: - KUBERNETES_CPU_LIMIT: 4 - KUBERNETES_CPU_REQUEST: 2 - KUBERNETES_MEMORY_LIMIT: 4Gi - script: - - rm -rf gromacs*tar.gz - - cd $BUILD_DIR - - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log - - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log - - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log - - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log - - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log - - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log - - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log - completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log - packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true - - grep "cannot be built" buildLogFile.log manBuildLogFile.log - completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log - packageBuildLogFile.log | tee -a buildErrors.log || true - - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi - - mv gromacs*tar.gz .. - - cd .. - # ugly hack begins here - - mkdir tempdir - - SOURCETARBALL=`ls gromacs*tar.gz` - - mv $SOURCETARBALL tempdir - - cd tempdir - - tar -xf $SOURCETARBALL - - rm -rf gromacs-*/$BUILD_DIR - - tar -c --gzip -f $SOURCETARBALL gromacs-*/ - - mv $SOURCETARBALL .. - - cd .. - - rm -rf tempdir - # Hack ends here - artifacts: - when: always - expire_in: 1 week - paths: - - $BUILD_DIR/*log - - gromacs*tar.gz - - -.simple-build-template: - # Docker images are from dockerhub, user eriklindahl - stage: pre-build - variables: - KUBERNETES_CPU_LIMIT: 8 - KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_LIMIT: 8Gi - CMAKE_COMPILER_SCRIPT: "" - CMAKE_EXTRA_OPTIONS: "" - CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" - CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" - CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" - CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" - CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" - only: - refs: - - web - - pushes - - schedules - - merge_requests - - script: - - echo $CMAKE_COMPILER_SCRIPT - - echo $CMAKE_EXTRA_OPTIONS - - echo $CMAKE_SIMD_OPTIONS - - echo $CMAKE_GPU_OPTIONS - - echo $CMAKE_MPI_OPTIONS - - echo $CMAKE_PRECISION_OPTIONS - - echo $CMAKE_BUILD_TYPE_OPTIONS - - echo $CMAKE_GMXAPI_OPTIONS - - if [[ -d $BUILD_DIR ]] ; then - echo "Cleaning up build directory" ; - rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; - else - echo "Preparing new build directory" ; - mkdir $BUILD_DIR ; - fi - - cd $BUILD_DIR - - cmake .. - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - $CMAKE_COMPILER_SCRIPT - $CMAKE_EXTRA_OPTIONS - $CMAKE_SIMD_OPTIONS - $CMAKE_MPI_OPTIONS - $CMAKE_PRECISION_OPTIONS - $CMAKE_BUILD_TYPE_OPTIONS - $CMAKE_GPU_OPTIONS - $CMAKE_GMXAPI_OPTIONS - -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON - 2>&1 | tee cmakeLog.log - - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - - cmake --build . -- -j8 2>&1 | tee buildLogFile.log - - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log - - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log - | grep -v "CMake" | tee buildErrors.log || true - - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true - - cmake --build . --target install 2>&1 | tee installBuildLogFile.log - - if [ -s buildErrors.log ] ; then echo "Found compiler warning during build"; cat buildErrors.log; exit 1; fi - - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true - - awk '/The following tests FAILED/,/^Errors while running CTest|^$/' - ctestLog.log | tee ctestErrors.log - - xsltproc scripts/CTest2JUnit.xsl Testing/`head -n 1 < Testing/TAG`/Test.xml > JUnitTestResults.xml - - if [ -s ctestErrors.log ] ; then - echo "Error during running ctest"; - exit 1; - fi - - cd .. - artifacts: - reports: - junit: $BUILD_DIR/JUnitTestResults.xml - paths: - - $BUILD_DIR/*log - when: always - expire_in: 1 week diff --git a/admin/ci-templates/.clang-before-script.yml b/admin/ci-templates/.clang-before-script.yml deleted file mode 100644 index 8161238658..0000000000 --- a/admin/ci-templates/.clang-before-script.yml +++ /dev/null @@ -1,14 +0,0 @@ -.clang-before-script-template: - variables: - COMPILER_MAJOR_VERSION: 7 - EXTRA_INSTALLS: "" - - before_script: - - echo $EXTRA_INSTALLS - - time apt-get -qq update - - time apt-get -qqy --no-install-suggests --no-install-recommends install - $EXTRA_INSTALLS - - mkdir -p ccache - - export CCACHE_BASEDIR=${PWD} - - export CCACHE_DIR=${PWD}/ccache - diff --git a/admin/ci-templates/.clang-build-template.yml b/admin/ci-templates/.clang-build-template.yml deleted file mode 100644 index 8ab7917892..0000000000 --- a/admin/ci-templates/.clang-build-template.yml +++ /dev/null @@ -1,6 +0,0 @@ -.build-clang-template: - variables: - DEBIAN_FRONTEND: noninteractive - COMPILER_MAJOR_VERSION: 7 - BUILD_DIR: build-clang - CMAKE_COMPILER_SCRIPT: -DCMAKE_C_COMPILER=clang-$COMPILER_MAJOR_VERSION -DCMAKE_CXX_COMPILER=clang++-$COMPILER_MAJOR_VERSION diff --git a/admin/ci-templates/.clang-test-template.yml b/admin/ci-templates/.clang-test-template.yml deleted file mode 100644 index c4c845dce3..0000000000 --- a/admin/ci-templates/.clang-test-template.yml +++ /dev/null @@ -1,4 +0,0 @@ -.test-clang-template: - variables: - COMPILER_MAJOR_VERSION: 7 - BUILD_DIR: build-clang diff --git a/admin/ci-templates/.cuda-template.yml b/admin/ci-templates/.cuda-template.yml deleted file mode 100644 index 20a89451b1..0000000000 --- a/admin/ci-templates/.cuda-template.yml +++ /dev/null @@ -1,8 +0,0 @@ -.cuda-template: - variables: - CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" - CUDA_BASE_VERSION: "" - BUILD_DIR: build-cuda - CMAKE_EXTRA_OPTIONS: -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda/ - CMAKE_GPU_OPTIONS: -DGMX_GPU=ON - diff --git a/admin/ci-templates/.documentation-before-script.yml b/admin/ci-templates/.documentation-before-script.yml deleted file mode 100644 index 43c73b4d52..0000000000 --- a/admin/ci-templates/.documentation-before-script.yml +++ /dev/null @@ -1,13 +0,0 @@ -.documentation-before-script-template: - variables: - EXTRA_INSTALLS: "" - - before_script: - - echo $EXTRA_INSTALLS - - time apt-get -qq update - - time apt-get -qqy --no-install-suggests --no-install-recommends install - $EXTRA_INSTALLS - - mkdir -p ccache - - export CCACHE_BASEDIR=${PWD} - - export CCACHE_DIR=${PWD}/ccache - diff --git a/admin/ci-templates/.gcc-before-script.yml b/admin/ci-templates/.gcc-before-script.yml deleted file mode 100644 index a3b2ac2a05..0000000000 --- a/admin/ci-templates/.gcc-before-script.yml +++ /dev/null @@ -1,14 +0,0 @@ -.gcc-before-script-template: - variables: - COMPILER_MAJOR_VERSION: 7 - EXTRA_INSTALLS: "" - - before_script: - - echo $EXTRA_INSTALLS - - time apt-get -qq update - - time apt-get -qqy --no-install-suggests --no-install-recommends install - $EXTRA_INSTALLS - - mkdir -p ccache - - export CCACHE_BASEDIR=${PWD} - - export CCACHE_DIR=${PWD}/ccache - diff --git a/admin/ci-templates/.gcc-build-template.yml b/admin/ci-templates/.gcc-build-template.yml deleted file mode 100644 index 8af8090274..0000000000 --- a/admin/ci-templates/.gcc-build-template.yml +++ /dev/null @@ -1,5 +0,0 @@ -.build-gcc-template: - variables: - COMPILER_MAJOR_VERSION: 7 - BUILD_DIR: build-gcc - CMAKE_COMPILER_SCRIPT: -DCMAKE_C_COMPILER=gcc-$COMPILER_MAJOR_VERSION -DCMAKE_CXX_COMPILER=g++-$COMPILER_MAJOR_VERSION diff --git a/admin/ci-templates/.gcc-test-template.yml b/admin/ci-templates/.gcc-test-template.yml deleted file mode 100644 index 1fb624317e..0000000000 --- a/admin/ci-templates/.gcc-test-template.yml +++ /dev/null @@ -1,4 +0,0 @@ -.test-gcc-template: - variables: - COMPILER_MAJOR_VERSION: 7 - BUILD_DIR: build-gcc diff --git a/admin/ci-templates/.image-templates.yml b/admin/ci-templates/.image-templates.yml deleted file mode 100644 index 311b45c46c..0000000000 --- a/admin/ci-templates/.image-templates.yml +++ /dev/null @@ -1,48 +0,0 @@ -.gcc5-template: - image: gromacs/ci-gcc-5:2020 - -.gcc6-template: - image: gromacs/ci-gcc-6:2020 - -.gcc7-template: - image: gromacs/ci-gcc-7:2020 - -.gcc8-template: - image: gromacs/ci-gcc-8:2020 - -.clang6-template: - image: gromacs/ci-clang-6:2020 - -.clang7-template: - image: gromacs/ci-clang-7:2020 - -.clang8-template: - image: gromacs/ci-clang-8:2020 - -.gcc5-gmxapi-template: - image: gromacs/ci-gcc-5-gmxapi:2020 - -.gcc6-gmxapi-template: - image: gromacs/ci-gcc-6-gmxapi:2020 - -.gcc7-gmxapi-template: - image: gromacs/ci-gcc-7-gmxapi:2020 - -.gcc8-gmxapi-template: - image: gromacs/ci-gcc-8-gmxapi:2020 - -.clang6-gmxapi-template: - image: gromacs/ci-clang-6-gmxapi:2020 - -.clang7-gmxapi-template: - image: gromacs/ci-clang-7-gmxapi:2020 - -.clang8-gmxapi-template: - image: gromacs/ci-clang-8-gmxapi:2020 - -.gcc-8-cuda-10.2-template: - image: gromacs/ci-gcc-8-cuda-10.2:2020 - variables: - CUDA_BASE_VERSION: "10.2" - COMPILER_MAJOR_VERSION: 8 - diff --git a/admin/ci-templates/.include-template.yml b/admin/ci-templates/.include-template.yml deleted file mode 100644 index 1027664ed1..0000000000 --- a/admin/ci-templates/.include-template.yml +++ /dev/null @@ -1,22 +0,0 @@ -include: - - local: '/admin/ci-templates/.image-templates.yml' - - local: '/admin/ci-templates/.build-cache-template.yml' - - local: '/admin/ci-templates/.pull-cache-template.yml' - - local: '/admin/ci-templates/.variables-template.yml' - - local: '/admin/ci-templates/.build-template.yml' - - local: '/admin/ci-templates/.test-runner-template.yml' - - local: '/admin/ci-templates/.gcc-before-script.yml' - - local: '/admin/ci-templates/.gcc-build-template.yml' - - local: '/admin/ci-templates/.gcc-test-template.yml' - - local: '/admin/ci-templates/.clang-build-template.yml' - - local: '/admin/ci-templates/.clang-test-template.yml' - - local: '/admin/ci-templates/.clang-before-script.yml' - - local: '/admin/ci-templates/.documentation-before-script.yml' - - local: '/admin/ci-templates/.tags-template.yml' - - local: '/admin/ci-templates/.test-script-template.yml' - - local: '/admin/ci-templates/.build-docs-template.yml' - - local: '/admin/ci-templates/.cuda-template.yml' - - local: '/admin/ci-templates/.build-mpi-template.yml' - - local: '/admin/ci-templates/.build-gmxapi-template.yml' - - local: '/admin/ci-templates/.regressiontest-template.yml' - - local: '/admin/ci-templates/.release-template.yml' diff --git a/admin/ci-templates/.pull-cache-template.yml b/admin/ci-templates/.pull-cache-template.yml deleted file mode 100644 index b35be1a138..0000000000 --- a/admin/ci-templates/.pull-cache-template.yml +++ /dev/null @@ -1,8 +0,0 @@ -.pull-cache-template: - cache: - key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - # Read but don't update the cache - policy: pull - -.no-cache-template: - cache: {} diff --git a/admin/ci-templates/.regressiontest-template.yml b/admin/ci-templates/.regressiontest-template.yml deleted file mode 100644 index 6641f9fa37..0000000000 --- a/admin/ci-templates/.regressiontest-template.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Set up to only fetch the files and prepare everything for merge requests -.regressiontests-template: - image: biophysics/gcc-gromacs - stage: configure-build - - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - only: - refs: - - web - - pushes - - merge_requests - - schedules -# Always clone the default version for this branch, release-2020 in this case - script: - - export REGTESTBRANCH=release-2020 - - if [[ ! -d regressiontests ]] ; then - mkdir regressiontests ; - cd regressiontests ; - git init ; - cd .. ; - fi - - cd regressiontests - - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH - - git checkout -qf FETCH_HEAD - - git clean -ffdxq - - git gc - - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD - - echo "Build regressiontests for branch $REGTESTBRANCH" - - mv gmx-regressiontests.tar.gz .. - - cd .. - artifacts: - paths: - - gmx-regressiontests.tar.gz - -.regressiontests-release-template: - image: biophysics/gcc-gromacs - stage: release-package - - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE - script: - - VERSION=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` - - REGTESTNAME="regressiontests-"$VERSION - - if [[ $GROMACS_RELEASE != "true" ]] ; then - REGTESTNAME=$REGTESTNAME-dev ; - fi - - export REGTESTBRANCH=release-2020 - - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then - export REGTESTBRANCH=$CI_COMMIT_REF_NAME ; - fi - - if [[ ! -d regressiontests ]] ; then - mkdir regressiontests ; - cd regressiontests ; - git init ; - cd .. ; - fi - - cd regressiontests - - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH - - git checkout -qf FETCH_HEAD - - git clean -ffdxq - - git gc - - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD - - mv $REGTESTNAME.tar.gz .. - - echo "Build regressiontests for branch $REGTESTBRANCH" - - cd .. - artifacts: - paths: - - regressiontests-*.tar.gz diff --git a/admin/ci-templates/.release-template.yml b/admin/ci-templates/.release-template.yml deleted file mode 100644 index f2a4b56328..0000000000 --- a/admin/ci-templates/.release-template.yml +++ /dev/null @@ -1,61 +0,0 @@ -.release-version-template: - image: biophysics/gcc-gromacs - stage: configure-build - - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - - only: - refs: - - merge_requests - - schedules - - web - variables: - - $GROMACS_RELEASE - script: - - cmake -P cmake/gmxVersionInfo.cmake &> version.json - - artifacts: - paths: - - version.json - - -.release-verify-template: - image: biophysics/gcc-gromacs - stage: release-verify - - variables: - KUBERNETES_CPU_LIMIT: 1 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi - - only: - refs: - - merge_requests - - schedules - - web - variables: - - $GROMACS_RELEASE - script: - - VERSION=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` - - if [[ $GROMACS_RELEASE != "true" ]] ; then - VERSION=$VERSION-dev ; - fi - - REGTEST_COMPARE=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"` - - SOURCENAME=gromacs-$VERSION - - SOURCETARBALL=$SOURCENAME.tar.gz - - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'` - - REGTESTNAME=regressiontests-$VERSION - - REGTESTTARBALL=$REGTESTNAME.tar.gz - - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'` - - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM" - - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM" - - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE" - - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then - echo "Mismatch in regressiontest md5sums"; - exit 1; - fi diff --git a/admin/ci-templates/.tags-template.yml b/admin/ci-templates/.tags-template.yml deleted file mode 100644 index 57d95802d1..0000000000 --- a/admin/ci-templates/.tags-template.yml +++ /dev/null @@ -1,3 +0,0 @@ -.tags-template: - tags: - - k8s-scilifelab diff --git a/admin/ci-templates/.test-runner-template.yml b/admin/ci-templates/.test-runner-template.yml deleted file mode 100644 index d5dec907f7..0000000000 --- a/admin/ci-templates/.test-runner-template.yml +++ /dev/null @@ -1,13 +0,0 @@ -.test-base-template: - variables: - BUILD_DIR: build - REGRESSIONTEST_TOTAL_RANK_NUMBER: 4 - REGRESSIONTEST_OMP_RANK_NUMBER: 2 - REGRESSIONTEST_PME_RANK_NUMBER: 1 - REGRESSIONTEST_MPI_RUN_COMMAND: "" - REGRESSIONTEST_DOUBLE: "" - image: biophysics/gcc-gromacs - stage: test - retry: - max: 2 - diff --git a/admin/ci-templates/.test-script-template.yml b/admin/ci-templates/.test-script-template.yml deleted file mode 100644 index fd056aadaa..0000000000 --- a/admin/ci-templates/.test-script-template.yml +++ /dev/null @@ -1,34 +0,0 @@ -.test-script-template: - only: - refs: - - web - - pushes - - schedules - - merge_requests - - script: - - cd $BUILD_DIR - - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true - - awk '/The following tests FAILED/,/^Errors while running CTest|^$/' - ctestLog.log | tee ctestErrors.log - - xsltproc scripts/CTest2JUnit.xsl Testing/`head -n 1 < Testing/TAG`/Test.xml > JUnitTestResults.xml - - if [ -s ctestErrors.log ] ; then - echo "Error during running ctest"; - exit 1; - fi - - cd .. - - tar xzf gmx-regressiontests.tar.gz - - source $INSTALL_DIR/bin/GMXRC - - cd regressiontests - - echo "perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND" - - perl gmxtest.pl -nt $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER - -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND -xml all - artifacts: - reports: - junit: $BUILD_DIR/JUnitTestResults.xml - paths: - - $BUILD_DIR/Testing - - $BUILD_DIR/tests - - regressiontests - when: always - expire_in: 1 week diff --git a/admin/ci-templates/.variables-template.yml b/admin/ci-templates/.variables-template.yml deleted file mode 100644 index b3d919a423..0000000000 --- a/admin/ci-templates/.variables-template.yml +++ /dev/null @@ -1,11 +0,0 @@ -.variables-template: - variables: - KUBERNETES_CPU_LIMIT: 8 - KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_LIMIT: 8Gi - KUBERNETES_EXTENDED_RESOURCE_NAME: "" - KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 - CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" - BUILD_DIR: build - INSTALL_DIR: install - diff --git a/admin/clang-tidy.sh b/admin/clang-tidy.sh index 7e75a98fce..6848787dab 100755 --- a/admin/clang-tidy.sh +++ b/admin/clang-tidy.sh @@ -42,8 +42,9 @@ # Parse command-line arguments function usage() { echo "usage: clang-tidy.sh [-f|--force] [--parallel=#Jobs] [--rev=REV]" - echo " [--format=(off|check)]" + echo " [--tidy=(off|check)]" echo " [--warnings=] []" + echo " [-B=]" echo ": (check*|diff|update)[-(index|workdir*)] (*=default)" } @@ -53,6 +54,7 @@ baserev="HEAD" force= tidy_mode=check warning_file= +builddir= concurrency=2 for arg in "$@" ; do if [[ "$arg" == "check-index" || "$arg" == "check-workdir" || \ @@ -64,8 +66,8 @@ for arg in "$@" ; do action=$arg-workdir elif [[ "$action" == diff-* ]] ; then diffargs+=("$arg") - elif [[ "$arg" == --format=* ]] ; then - tidy_mode=${arg#--format=} + elif [[ "$arg" == --tidy=* ]] ; then + tidy_mode=${arg#--tidy=} if [[ "$tidy_mode" != "off" && "$tidy_mode" != "check" ]] ; then echo "Unknown option: $arg" echo @@ -74,12 +76,14 @@ for arg in "$@" ; do fi elif [[ "$arg" == "-f" || "$arg" == "--force" ]] ; then force=1 - elif [[ "$arg" == "--parallel=*" ]] ; then + elif [[ "$arg" == --parallel=* ]] ; then concurrency=${arg#--parallel=} elif [[ "$arg" == --rev=* ]] ; then baserev=${arg#--rev=} elif [[ "$arg" == --warnings=* ]] ; then warning_file=${arg#--warnings=} + elif [[ "$arg" == -B=* ]] ; then + builddir=${arg#-B=} elif [[ "$arg" == "-h" || "$arg" == "--help" ]] ; then usage exit 0 @@ -96,27 +100,26 @@ if [[ "$tidy_mode" != "off" ]] then if [ -z "$RUN_CLANG_TIDY" ] then - RUN_CLANG_TIDY=`git config hooks.run_clang_tidypath` + RUN_CLANG_TIDY=`git config hooks.runclangtidypath` fi if [ -z "$RUN_CLANG_TIDY" ] then echo "Please set the path to run-clang-tidy using the git hook" - echo "git config hooks.run_clang_tidypath /path/to/run-clang-tidy" + echo "git config hooks.runclangtidypath /path/to/run-clang-tidy-8.py" echo "or by setting an environment variable, e.g." - echo "RUN_CLANG_TIDY=/path/to/run-clang-tidy" + echo "RUN_CLANG_TIDY=/path/to/run-clang-tidy-8.py" exit 2 fi if ! which "$RUN_CLANG_TIDY" 1>/dev/null then - echo "run-clang-tidy not found: $RUN_CLANG_TIDY" + echo "run-clang-tidy-8.py not found: $RUN_CLANG_TIDY" exit 2 fi fi # Switch to the root of the source tree and check the config file srcdir=`git rev-parse --show-toplevel` -pushd $srcdir >/dev/null -admin_dir=$srcdir/admin +pushd $srcdir >/dev/null || exit # Actual processing starts: create a temporary directory tmpdir=`mktemp -d -t gmxclangtidy.XXXXXX` @@ -133,23 +136,33 @@ cut -f2 <$tmpdir/difflist | \ git check-attr --stdin filter | \ sed -e 's/.*: filter: //' | \ paste $tmpdir/difflist - | \ - grep -E '(complete_formatting)$' >$tmpdir/filtered + grep -E '(complete_formatting|clangformat|copyright|includesort)$' >$tmpdir/filtered cut -f2 <$tmpdir/filtered >$tmpdir/filelist_all -grep -E '(complete_formatting)$' <$tmpdir/filtered | \ +grep -E '(complete_formatting|clangformat)$' <$tmpdir/filtered | \ cut -f2 >$tmpdir/filelist_clangtidy git diff-files --name-only | grep -Ff $tmpdir/filelist_all >$tmpdir/localmods # Extract changed files to a temporary directory mkdir $tmpdir/org if [[ $action == *-index ]] ; then - git checkout-index --prefix=$tmpdir/org/ + git checkout-index --prefix=$tmpdir/org/ --stdin <$tmpdir/filelist_all else - rsync -a $srcdir/src/ $tmpdir/org/src/ + rsync --files-from=$tmpdir/filelist_all -a $srcdir/ $tmpdir/org/ fi -# Need to have compilation database file available somewhere above where we are using it -rsync -a $srcdir/compile_commands.json $tmpdir/org -# Duplicate the original files to a separate directory, where all changes will -# be made. +# check for the existence of the compile_commands.json file and abort +# if it is not present. If we don't have a build directory, try the +# current source directory. +if [ -z $builddir ] ; then + builddir=$srcdir +fi +if [[ ! -f $builddir/compile_commands.json ]] ; then + echo "Could not find compile_commands.json in builddir=$builddir" + echo "Make sure you gave a correct build tree and that it contains the file!" +else + # Need to have compilation database file available somewhere above where we are using it + rsync -a $builddir/compile_commands.json $tmpdir/org +fi +# Prepare directory to use for comparing changed and original files cp -r $tmpdir/org $tmpdir/new # Create output file for what was done (in case no messages get written) @@ -159,20 +172,23 @@ touch $tmpdir/messages # Can only perform clang-tidy on a non-empty list of files cd $tmpdir/new if [[ $tidy_mode != "off" && -s $tmpdir/filelist_clangtidy ]] ; then - $RUN_CLANG_TIDY `cat $tmpdir/filelist_clangtidy` -- -header-filter=.* -j $concurrency -fix -fix-errors --cuda-host-only -nocudainc -quiet >$tmpdir/clang-tidy.out 2>&1 - grep -v "clang-analyzer" $tmpdir/clang-tidy.out | grep -v "to display errors from all non" | grep -i "error|warning" - > $tmpdir/clang-tidy-errors.out + $RUN_CLANG_TIDY `cat $tmpdir/filelist_clangtidy` -header-filter=.* -j $concurrency -fix -quiet -extra-arg=--cuda-host-only -extra-arg=-nocudainc>$tmpdir/clang-tidy.out 2>&1 + awk '/warning/,/clang-tidy|^$/' $tmpdir/clang-tidy.out | grep -v "warnings generated." | grep -v "Suppressed .* warnings" | grep -v "clang-analyzer" | grep -v "to display errors from all non" | sed '/^\s*$/d' > $tmpdir/clang-tidy-warnings.out + awk '/.*error.*/' $tmpdir/clang-tidy.out > $tmpdir/clang-tidy-errors.out || true if [ -s $tmpdir/clang-tidy-errors.out ]; then - echo "Running code tidying failed. Check output below for errors:" + echo "Running of clang-tidy failed. Check output below for errors:" cat $tmpdir/clang-tidy-errors.out rm -rf $tmpdir exit 2 fi # Find the changed files if necessary if [[ $action != diff-* ]] ; then - msg="needs formatting" + msg="found code issues" if [[ $action == update-* ]] ; then msg="clang-tidy performed" fi + rsync --files-from=$tmpdir/filelist_all -a $srcdir/ ./ + rsync -a $tmpdir/org/ $srcdir/ git diff --no-index --name-only ../org/ . | \ awk -v msg="$msg" '{sub(/.\//,""); print $0 ": " msg}' >> $tmpdir/messages fi @@ -240,7 +256,10 @@ fi popd >/dev/null # Report what was done -sort $tmpdir/messages | tee $warning_file +if [ -s $tmpdir/clang-tidy-warnings.out ] ; then + cat $tmpdir/clang-tidy-warnings.out | tee $warning_file +fi +sort $tmpdir/messages | tee -a $warning_file rm -rf $tmpdir exit $changes diff --git a/admin/containers/buildall.sh b/admin/containers/buildall.sh new file mode 100644 index 0000000000..1492f0f775 --- /dev/null +++ b/admin/containers/buildall.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -ev + +SCRIPT=$PWD/scripted_gmx_docker_builds.py + +# Note: All official GROMACS CI images are built +# with openmpi on. That reduces the total number of +# images needed, because the same one can test library, +# thread and no MPI configurations. + +tag="gromacs/cmake-3.9.6-gcc-5-cuda-9.0-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.9.6 --gcc 5 --cuda 9.0 --ubuntu 16.04 --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.9.6 --gcc 6 --cuda 10.1 --opencl --clfft --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.9.6 --gcc 7 --opencl amd --clfft --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.15.7-gcc-8-cuda-10.1-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.15.7 --gcc 8 --cuda 10.1 --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.9.6-gcc-9-cuda-10.0-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.9.6 --gcc 9 --cuda 10.0 --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.11.4-llvm-8-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.11.4 --llvm 8 --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.15.7-llvm-8-tsan:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.15.7 --llvm 8 --tsan | docker build -t $tag - + +tag="gromacs/cmake-3.15.7-llvm-8-cuda-10.1-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.15.7 --llvm 8 --cuda 10.1 --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --cmake 3.15.7 --llvm 8 --opencl intel --mpi openmpi | docker build -t $tag - + +tag="gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020" +tags[${#tags[@]}]=$tag +python3 $SCRIPT --ubuntu 16.04 --cmake 3.9.6 --llvm 3.6 --opencl amd --mpi openmpi | docker build -t $tag - + +tag=gromacs/ci-docs-llvm:2020 +tags[${#tags[@]}]=$tag +python3 $SCRIPT --llvm --doxygen | docker build -t $tag - + +tag=gromacs/ci-docs-gcc:2020 +tags[${#tags[@]}]=$tag +python3 $SCRIPT --gcc --doxygen | docker build -t $tag - + +docker login +for tag in "${tags[@]}"; do + echo "Pushing $tag" + #docker push $tag +done diff --git a/admin/containers/scripted_gmx_docker_builds.py b/admin/containers/scripted_gmx_docker_builds.py index 9046b27fe3..1efd05b5dc 100644 --- a/admin/containers/scripted_gmx_docker_builds.py +++ b/admin/containers/scripted_gmx_docker_builds.py @@ -58,9 +58,13 @@ Usage:: """ import argparse +import collections +import typing +from distutils.version import StrictVersion import hpccm import hpccm.config +from hpccm.building_blocks.base import bb_base try: import utility @@ -69,6 +73,68 @@ except ImportError: 'This module assumes availability of supporting modules in the same directory. Add the directory to ' 'PYTHONPATH or invoke Python from within the module directory so module location can be resolved.') +# Basic packages for all final images. +_common_packages = ['build-essential', + 'ca-certificates', + 'ccache', + 'git', + 'gnupg', + 'libfftw3-dev', + 'libhwloc-dev', + 'liblapack-dev', + 'libx11-dev', + 'moreutils', + 'ninja-build', + 'rsync', + 'valgrind', + 'vim', + 'wget', + 'xsltproc'] + +# Extra packages needed to build Python installations from source. +_python_extra_packages = ['build-essential', + 'ca-certificates', + 'ccache', + 'curl', + 'git', + 'libbz2-dev', + 'libffi-dev', + 'liblzma-dev', + 'libncurses5-dev', + 'libncursesw5-dev', + 'libreadline-dev', + 'libsqlite3-dev', + 'libssl-dev', + 'llvm', + 'python-openssl', + 'vim', + 'wget', + 'zlib1g-dev'] + +# Extra packages needed for images for building documentation. +_docs_extra_packages = ['autoconf', + 'automake', + 'autopoint', + 'autotools-dev', + 'bison', + 'flex', + 'ghostscript', + 'graphviz', + 'help2man', + 'imagemagick', + 'libtool', + 'linkchecker', + 'mscgen', + 'm4', + 'texinfo', + 'texlive-latex-base', + 'texlive-latex-extra', + 'texlive-fonts-recommended', + 'texlive-fonts-extra'] + +# Supported Python versions for maintained branches. +# TODO: Remove '3.5.9' from defaults in master once script in release-2020 diverges. +_python_versions = ['3.5.9', '3.6.10', '3.7.7', '3.8.2'] # Parse command line arguments parser = argparse.ArgumentParser(description='GROMACS CI image creation script', parents=[utility.parser]) @@ -76,195 +142,366 @@ parser = argparse.ArgumentParser(description='GROMACS CI image creation script', parser.add_argument('--format', type=str, default='docker', choices=['docker', 'singularity'], help='Container specification format (default: docker)') +parser.add_argument('--venvs', nargs='*', type=str, default=_python_versions, + help='List of Python versions ("major.minor.patch") for which to install venvs. ' + 'Default: {}'.format(' '.join(_python_versions))) -def main(args) -> hpccm.Stage: - # Create Stage - Stage0 = hpccm.Stage() - - # Create string for base image tag - base_image_tag = str() +def base_image_tag(args) -> str: # Check if we use CUDA images or plain linux images - if (args.cuda is not None): + if args.cuda is not None: cuda_version_tag = 'nvidia/cuda:' + args.cuda + '-devel' - if (args.centos is not None): + if args.centos is not None: cuda_version_tag += '-centos' + args.centos - elif (args.ubuntu is not None): - if ((args.cuda == '9.0') and (args.ubuntu == '18.04')): - raise RuntimeError('Can not combine CUDA 9.0 and Ubuntu 18.04') + elif args.ubuntu is not None: cuda_version_tag += '-ubuntu' + args.ubuntu else: raise RuntimeError('Logic error: no Linux distribution selected.') base_image_tag = cuda_version_tag else: - if (args.centos is not None): + if args.centos is not None: base_image_tag = 'centos:centos' + args.centos - elif (args.ubuntu is not None): + elif args.ubuntu is not None: base_image_tag = 'ubuntu:' + args.ubuntu else: raise RuntimeError('Logic error: no Linux distribution selected.') + return base_image_tag - Stage0 += hpccm.primitives.baseimage(image=base_image_tag) - - # Install the GROMACS packages we always will need for our builds. - Stage0 += hpccm.building_blocks.packages(ospackages=['build-essential', - 'ccache', - 'git', - 'libfftw3-dev', - 'libhwloc-dev', - 'liblapack-dev', - 'libx11-dev', - 'moreutils', - 'ninja-build', - 'rsync', - 'valgrind', - 'wget', - 'xsltproc']) - - # Add CMake to image - Stage0 += hpccm.building_blocks.cmake(eula=True, version=args.cmake) - # We always add Python3 and Pip - Stage0 += hpccm.building_blocks.python(python3=True, python2=False, devel=True) - Stage0 += hpccm.building_blocks.pip(upgrade=True, pip='pip3', - packages=['pytest', 'networkx', 'numpy']) +def get_llvm_packages(args) -> typing.Iterable[str]: + # If we use the package version of LLVM, we need to install extra packages for it. + if (args.llvm is not None) and (args.tsan is None): + return ['libomp-dev', + 'clang-format-' + str(args.llvm), + 'clang-tidy-' + str(args.llvm)] + else: + return [] + +def get_compiler(args, tsan_stage: hpccm.Stage = None) -> bb_base: # Compiler - if (args.icc is not None): + if args.icc is not None: raise RuntimeError('Intel compiler toolchain recipe not implemented yet') - if (args.llvm is not None): - # Build the default compiler if we don't need special support - if (args.tsan is None): - if (args.llvm == 3): - if ((args.ubuntu is not None) and (args.ubuntu == '18.04')): - raise RuntimeError('LLVM 3 and Ubuntu 18.04 can cause issues when used together') - args.llvm = 3.6 - compiler = hpccm.building_blocks.llvm(extra_repository=True, version=args.llvm) + if args.llvm is not None: # Build our own version instead to get TSAN + OMP + if args.tsan is not None: + if tsan_stage is not None: + compiler = tsan_stage.runtime(_from='tsan') + else: + raise RuntimeError('No TSAN stage!') + # Build the default compiler if we don't need special support else: - compiler_branch = 'release_'+str(args.llvm)+'0' - compiler = hpccm.building_blocks.generic_cmake(repository='https://git.llvm.org/git/llvm.git', - prefix='/usr/local', recursive=True, branch=compiler_branch, - cmake_opts=['-D CMAKE_BUILD_TYPE=Release', '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra"', '-D LIBOMP_TSAN_SUPPORT=on'], - preconfigure=['export branch='+compiler_branch, - '(cd projects; git clone https://git.llvm.org/git/libcxx.git; cd libcxx; git checkout $branch)', - '(cd projects; git clone https://git.llvm.org/git/libcxxabi.git; cd libcxxabi; git checkout $branch)', - '(cd projects; git clone https://git.llvm.org/git/compiler-rt.git; cd compiler-rt; git checkout $branch)', - '(cd ..; git clone https://git.llvm.org/git/openmp.git; cd openmp; git checkout $branch)', - '(cd ..; git clone https://git.llvm.org/git/clang.git; cd clang; git checkout $branch)', - '(cd ..; git clone https://git.llvm.org/git/clang-tools-extra.git clang-tools-extra; cd clang-tools-extra; git checkout $branch)'], - postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-'+str(args.llvm), - 'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-'+str(args.llvm), - 'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-'+str(args.llvm), - 'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-'+str(args.llvm)]) - - - elif (args.gnu is not None): + compiler = hpccm.building_blocks.llvm(extra_repository=True, version=args.llvm) + + elif (args.gcc is not None): compiler = hpccm.building_blocks.gnu(extra_repository=True, - version=args.gnu, + version=args.gcc, fortran=False) else: raise RuntimeError('Logic error: no compiler toolchain selected.') + return compiler - Stage0 += compiler - # If we use the package version of LLVM, we need to install extra packages for it. - if (args.llvm is not None) and (args.tsan is None): - Stage0 += hpccm.building_blocks.packages(ospackages=['libomp-dev', - 'clang-format-'+str(args.llvm), - 'clang-tidy-'+str(args.llvm)]) - +def get_mpi(args, compiler): # If needed, add MPI to the image - if (args.mpi is not None): + if args.mpi is not None: if args.mpi == 'openmpi': use_cuda = False - if (args.cuda is not None): + if args.cuda is not None: use_cuda = True - Stage0 += hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, cuda=use_cuda, infiniband=False) + if hasattr(compiler, 'toolchain'): + return hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, cuda=use_cuda, infiniband=False) + else: + raise RuntimeError('compiler is not an HPCCM compiler building block!') + elif args.mpi == 'impi': raise RuntimeError('Intel MPI recipe not implemented yet.') + else: + raise RuntimeError('Requested unknown MPI implementation.') + else: + return None + +def get_opencl(args): # Add OpenCL environment if needed if (args.opencl is not None): if args.opencl == 'nvidia': if (args.cuda is None): raise RuntimeError('Need Nvidia environment for Nvidia OpenCL image') - Stage0 += hpccm.building_blocks.packages(ospackages=['nvidia-opencl-dev']) + return hpccm.building_blocks.packages(ospackages=['nvidia-opencl-dev']) elif args.opencl == 'intel': - Stage0 += hpccm.building_blocks.packages(ospackages=['ocl-icd-opencl-dev', 'opencl-headers', 'beignet-opencl-icd']) + return hpccm.building_blocks.packages( + apt_ppas=['ppa:intel-opencl/intel-opencl'], + ospackages=['opencl-headers', 'ocl-icd-libopencl1', + 'ocl-icd-opencl-dev', 'intel-opencl-icd']) + elif args.opencl == 'amd': - # Due to the wisdom of AMD, this needs to be done differently for the OS and version! Hurray! - # And they don't allow wget, so this branch is not taken for now! AMD, please allow me to use wget. - raise RuntimeError('AMD recipe can not be generated because they do not allow wget for getting the packages.') - # if args.ubuntu: - # if args.ubuntu is not '16.04': - # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/ubuntu/'+args.ubuntu+'/amdgpu-pro-18.30-641594.tar.xz', - # install=['./amdgpu-install --opencl=legacy --headless -y']) - # elif: - # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/ubuntu/amdgpu-pro-18.30-641594.tar.xz', - # install=['./amdgpu-install --opencl=legacy --headless -y']) - # elif args.centos: - # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/rhel'+args.centos'/amdgpu-pro-18.30-641594.tar.xz', - # install=['./amdgpu-install --opencl=legacy --headless -y']) - - if (args.clfft is not None): - Stage0 += hpccm.building_blocks.generic_cmake(repository='https://github.com/clMathLibraries/clFFT.git', - prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src') + # libelf1 is a necessary dependency for something in the ROCm stack, + # which they should set up, but seem to have omitted. + return hpccm.building_blocks.packages( + apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'], + apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main'], + ospackages=['ocl-icd-libopencl1', 'ocl-icd-opencl-dev', 'opencl-headers', 'libelf1', 'rocm-opencl']) + else: + return None + + +def get_clfft(args): + if (args.clfft is not None): + return hpccm.building_blocks.generic_cmake( + repository='https://github.com/clMathLibraries/clFFT.git', + prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src') + else: + return None + + +def add_tsan_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]): + """Isolate the expensive TSAN preparation stage. + + This is a very expensive stage, but has few and disjoint dependencies, and + its output is easily compartmentalized (/usr/local) so we can isolate this + build stage to maximize build cache hits and reduce rebuild time, bookkeeping, + and final image size. + """ + if not isinstance(output_stages, collections.abc.MutableMapping): + raise RuntimeError('Need output_stages container.') + tsan_stage = hpccm.Stage() + tsan_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='tsan') + + tsan_stage += hpccm.building_blocks.packages(ospackages=['git', 'ca-certificates', 'build-essential', 'cmake']) + # CMake will get duplicated later, but this is an expensive image, and it isn't worth optimizing + # out that duplication... + tsan_stage += hpccm.building_blocks.python(python3=True, python2=False, devel=False) + + compiler_branch = 'release_' + str(input_args.llvm) + '0' + tsan_stage += hpccm.building_blocks.generic_cmake( + repository='https://git.llvm.org/git/llvm.git', + prefix='/usr/local', recursive=True, branch=compiler_branch, + cmake_opts=['-D CMAKE_BUILD_TYPE=Release', '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra"', + '-D LIBOMP_TSAN_SUPPORT=on'], + preconfigure=['export branch=' + compiler_branch, + '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxx.git)', + '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxxabi.git)', + '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/compiler-rt.git)', + '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/openmp.git)', + '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang.git)', + '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang-tools-extra.git)'], + postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-' + str(input_args.llvm), + 'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-' + str(input_args.llvm), + 'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-' + str(input_args.llvm), + 'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-' + str(input_args.llvm)]) + output_stages['tsan'] = tsan_stage + + +def prepare_venv(version: StrictVersion) -> typing.Sequence[str]: + """Get shell commands to set up the venv for the requested Python version.""" + major = version.version[0] + minor = version.version[1] + + pyenv = '$HOME/.pyenv/bin/pyenv' + + py_ver = '{}.{}'.format(major, minor) + venv_path = '$HOME/venv/py{}'.format(py_ver) + commands = ['$({pyenv} prefix `{pyenv} whence python{py_ver}`)/bin/python -m venv {path}'.format( + pyenv=pyenv, + py_ver=py_ver, + path=venv_path + )] + + commands.append('{path}/bin/python -m pip install --upgrade pip setuptools'.format( + path=venv_path + )) + # Install dependencies for building and testing gmxapi Python package. + # WARNING: Please keep this list synchronized with python_packaging/requirements-test.txt + # TODO: Get requirements.txt from an input argument. + commands.append("""{path}/bin/python -m pip install --upgrade \ + 'cmake>=3.9.6' \ + 'flake8>=3.7.7' \ + 'mpi4py>=2' \ + 'networkx>=2.0' \ + 'numpy>=1' \ + 'pip>=10.1' \ + 'pytest>=3.9' \ + 'setuptools>=28.0.0' \ + 'scikit-build>=0.7'""".format(path=venv_path)) + + return commands + + +def add_python_stages(building_blocks: typing.Mapping[str, bb_base], + input_args, + output_stages: typing.MutableMapping[str, hpccm.Stage]): + """Add the stage(s) necessary for the requested venvs. + + One intermediate build stage is created for each venv (see --venv option). + + Each stage partially populates Python installations and venvs in the home + directory. The home directory is collected by the 'pyenv' stage for use by + the main build stage. + """ + if len(input_args.venvs) < 1: + raise RuntimeError('No venvs to build...') + if output_stages is None or not isinstance(output_stages, collections.abc.Mapping): + raise RuntimeError('Need a container for output stages.') + + # Main Python stage that collects the environments from individual stages. + # We collect the stages individually, rather than chaining them, because the + # copy is a bit slow and wastes local Docker image space for each filesystem + # layer. + pyenv_stage = hpccm.Stage() + pyenv_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='pyenv') + pyenv_stage += building_blocks['compiler'] + pyenv_stage += building_blocks['mpi'] + pyenv_stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages) + + for version in [StrictVersion(py_ver) for py_ver in sorted(input_args.venvs)]: + stage_name = 'py' + str(version) + stage = hpccm.Stage() + stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as=stage_name) + stage += building_blocks['compiler'] + stage += building_blocks['mpi'] + stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages) + + # TODO: Use a non-root user for testing and Python virtual environments. + stage += hpccm.primitives.shell(commands=[ + 'curl https://pyenv.run | bash', + """echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $HOME/.bashrc""", + """echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> $HOME/.bashrc""", + """echo 'eval "$(pyenv init -)"' >> $HOME/.bashrc""", + """echo 'eval "$(pyenv virtualenv-init -)"' >> $HOME/.bashrc"""]) + pyenv = '$HOME/.pyenv/bin/pyenv' + commands = ['PYTHON_CONFIGURE_OPTS="--enable-shared" {pyenv} install -s {version}'.format( + pyenv=pyenv, + version=str(version))] + stage += hpccm.primitives.shell(commands=commands) + + commands = prepare_venv(version) + stage += hpccm.primitives.shell(commands=commands) + + # TODO: Update user home directory. + pyenv_stage += hpccm.primitives.copy(_from=stage_name, _mkdir=True, src=['/root/'], + dest='/root') + + # Add the intermediate build stage to the sequence + output_stages[stage_name] = stage + + # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured. + # # `version` should be a system installation or pyenv environment (or pyenv-virtualenv) + # # with the dependencies for all of the Python aspects of CMake-driven builds. + # commands = '{pyenv} global {version}'.format( + # pyenv=pyenv, + # version=...) + # pyenv_stage += hpccm.primitives.shell(commands=commands) + + # Add the aggregating build stage to the sequence. This allows the main stage to copy + # the files in a single stage, potentially reducing the overall output image size. + output_stages['pyenv'] = pyenv_stage + + +def build_stages(args) -> typing.Iterable[hpccm.Stage]: + """Define and sequence the stages for the recipe corresponding to *args*.""" + + # A Dockerfile or Singularity recipe can have multiple build stages. + # The main build stage can copy files from previous stages, though only + # the last stage is included in the tagged output image. This means that + # large or expensive sets of build instructions can be isolated in + # local/temporary images, but all of the stages need to be output by this + # script, and need to occur in the correct order, so we create a sequence + # object early in this function. + stages = collections.OrderedDict() + + # If we need the TSAN compilers, the early build is more involved. + if args.llvm is not None and args.tsan is not None: + add_tsan_stage(input_args=args, output_stages=stages) + + # Building blocks are chunks of container-builder instructions that can be + # copied to any build stage with the addition operator. + building_blocks = collections.OrderedDict() + + # These are the most expensive and most reusable layers, so we put them first. + building_blocks['compiler'] = get_compiler(args, tsan_stage=stages.get('tsan')) + building_blocks['mpi'] = get_mpi(args, building_blocks['compiler']) + + # Install additional packages early in the build to optimize Docker build layer cache. + os_packages = _common_packages + get_llvm_packages(args) + if args.doxygen is not None: + os_packages += _docs_extra_packages + building_blocks['ospackages'] = hpccm.building_blocks.packages(ospackages=os_packages) + + building_blocks['cmake'] = hpccm.building_blocks.cmake(eula=True, version=args.cmake) + building_blocks['opencl'] = get_opencl(args) + building_blocks['clfft'] = get_clfft(args) + + # Add Python environments to MPI images, only, so we don't have to worry + # about whether to install mpi4py. + if args.mpi is not None and len(args.venvs) > 0: + add_python_stages(building_blocks=building_blocks, input_args=args, output_stages=stages) + + # Create the stage from which the targeted image will be tagged. + stages['main'] = hpccm.Stage() + + stages['main'] += hpccm.primitives.baseimage(image=base_image_tag(args)) + for bb in building_blocks.values(): + if bb is not None: + stages['main'] += bb + + # We always add Python3 and Pip + stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True) + stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3', + packages=['pytest', 'networkx', 'numpy']) # Add documentation requirements (doxygen and sphinx + misc). if (args.doxygen is not None): if (args.doxygen == '1.8.5'): - doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a' - elif (args.doxygen == '1.8.11'): - doxygen_commit = 'a6d4f4df45febe588c38de37641513fd576b998f' + doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a' else: - raise RuntimeError('Need to provide either 1.8.5 or 1.8.11 as doxygen version.') - Stage0 += hpccm.building_blocks.packages(ospackages=['autoconf', - 'automake', - 'autopoint', - 'autotools-dev', - 'bison', - 'flex', - 'ghostscript', - 'graphviz', - 'help2man', - 'imagemagick', - 'libtool', - 'linkchecker', - 'mscgen', - 'm4', - 'texinfo', - 'texlive-latex-base', - 'texlive-latex-extra', - 'texlive-fonts-recommended', - 'texlive-fonts-extra']) - Stage0 += hpccm.building_blocks.generic_autotools(repository='https://github.com/westes/flex.git', - commit='f7788a9a0ecccdc953ed12043ccb59ca25714018', - prefix='/tmp/install-of-flex', - configure_opts=['--disable-shared'], - preconfigure=['./autogen.sh']) - Stage0 += hpccm.building_blocks.generic_autotools(repository='https://github.com/doxygen/doxygen.git', - commit=doxygen_commit, - prefix='', - configure_opts=['--flex /tmp/install-of-flex/bin/flex', '--static'], - postinstall=['sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml']) - Stage0 += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1']) - - return Stage0 + doxygen_commit = 'a6d4f4df45febe588c38de37641513fd576b998f' + stages['main'] += hpccm.building_blocks.generic_autotools( + repository='https://github.com/westes/flex.git', + commit='f7788a9a0ecccdc953ed12043ccb59ca25714018', + prefix='/tmp/install-of-flex', + configure_opts=['--disable-shared'], + preconfigure=['./autogen.sh']) + stages['main'] += hpccm.building_blocks.generic_autotools( + repository='https://github.com/doxygen/doxygen.git', + commit=doxygen_commit, + prefix='', + configure_opts=[ + '--flex /tmp/install-of-flex/bin/flex', + '--static'], + postinstall=[ + 'sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml']) + stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1']) + + if 'pyenv' in stages and stages['pyenv'] is not None: + stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/.pyenv/'], + dest='/root/.pyenv') + stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/venv/'], + dest='/root/venv') + # TODO: Update user home directory. + # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured. + # stages['main'] += hpccm.primitives.copy(_from='pyenv', src=['/root/.bashrc'], + # dest='/root/') + + # Note that the list of stages should be sorted in dependency order. + for build_stage in stages.values(): + if build_stage is not None: + yield build_stage if __name__ == '__main__': args = parser.parse_args() - container_recipe = main(args) - # Set container specification output format hpccm.config.set_container_format(args.format) + container_recipe = build_stages(args) + # Output container specification - print(container_recipe) + for stage in container_recipe: + print(stage) diff --git a/admin/containers/utility.py b/admin/containers/utility.py index e612f7b628..473881e4af 100644 --- a/admin/containers/utility.py +++ b/admin/containers/utility.py @@ -61,12 +61,12 @@ parser.add_argument('--cmake', type=str, default='3.9.6', choices=['3.9.6', '3.11.4', '3.15.7'], help='Selection of CMake version to provide to base image') compiler_group = parser.add_mutually_exclusive_group() -compiler_group.add_argument('--gnu', type=int, nargs='?', const=7, default=7, +compiler_group.add_argument('--gcc', type=int, nargs='?', const=7, default=7, choices=[5, 6, 7, 8, 9], help='Select GNU compiler tool chain. (Default) ' 'Some checking is implemented to avoid incompatible combinations') -compiler_group.add_argument('--llvm', type=int, nargs='?', const=7, default=None, - choices=[3, 6, 7, 8], +compiler_group.add_argument('--llvm', type=str, nargs='?', const='7', default=None, + choices=['3.6', '6', '7', '8'], help='Select LLVM compiler tool chain. ' 'Some checking is implemented to avoid incompatible combinations') compiler_group.add_argument('--icc', type=int, nargs='?', const=19, default=None, diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml new file mode 100644 index 0000000000..0d4901cfbe --- /dev/null +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -0,0 +1,280 @@ +# Packages, exported artifacts, and release engineering processes. + +prepare-release-version: + extends: + - .variables:default + - .rules:nightly-only-for-release + cache: {} + # Docker image uploaded to dockerhub by user eriklindahl + # TODO: Get DockerFile for admin/dockerfiles + image: biophysics/gcc-gromacs + stage: configure-build + + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + script: + - cmake -P cmake/gmxVersionInfo.cmake &> version.json + + artifacts: + paths: + - version.json + +# Special job to package regressiontest files and have them available for testing +# Runs during pre-build +# Set up to only fetch the files and prepare everything for merge requests +regressiontests:prepare: + extends: + - .variables:default + - .rules:merge-requests + cache: {} + image: biophysics/gcc-gromacs + stage: configure-build + + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + # Always clone the default version for this branch, master in this case + script: + - export REGTESTBRANCH=release-2020 + - if [[ ! -d regressiontests ]] ; then + mkdir regressiontests ; + cd regressiontests ; + git init ; + cd .. ; + fi + - cd regressiontests + - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH + - git checkout -qf FETCH_HEAD + - git clean -ffdxq + - git gc + - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD + - echo "Build regressiontests for branch $REGTESTBRANCH" + - mv gmx-regressiontests.tar.gz .. + - cd .. + artifacts: + paths: + - gmx-regressiontests.tar.gz + +regressiontests:package: + extends: + - .variables:default + - .rules:nightly-only-for-release + cache: {} + # Docker image uploaded to dockerhub by user eriklindahl + # TODO: Get DockerFile for admin/dockerfiles + image: biophysics/gcc-gromacs + stage: release-package + + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + needs: + - job: prepare-release-version + script: + - VERSION=`cat version.json | + python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` + - REGTESTNAME="regressiontests-"$VERSION + - if [[ $GROMACS_RELEASE != "true" ]] ; then + REGTESTNAME=$REGTESTNAME-dev ; + fi + - export REGTESTBRANCH=release-2020 + - if [[ $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-20"[1-2][0-9] ]] ; then + export REGTESTBRANCH=$CI_COMMIT_REF_NAME ; + fi + - if [[ ! -d regressiontests ]] ; then + mkdir regressiontests ; + cd regressiontests ; + git init ; + cd .. ; + fi + - cd regressiontests + - git fetch https://gitlab.com/gromacs/gromacs-regressiontests.git $REGTESTBRANCH + - git checkout -qf FETCH_HEAD + - git clean -ffdxq + - git gc + - git archive -o $REGTESTNAME.tar.gz --prefix $REGTESTNAME/ -9 HEAD + - git archive -o gmx-regressiontests.tar.gz --prefix regressiontests/ -9 HEAD + - mv $REGTESTNAME.tar.gz gmx-regressiontests.tar.gz .. + - echo "Build regressiontests for branch $REGTESTBRANCH" + - cd .. + artifacts: + paths: + - regressiontests-*.tar.gz + - gmx-regressiontests.tar.gz + +archive:configure:nightly: + extends: + - .docs:configure + - .rules:nightly-not-for-release + stage: nightly-configure + dependencies: [] + variables: + BUILD_DIR: build-package + CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None + CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF + CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF + +archive:configure:release: + extends: + - .docs:configure + - .rules:nightly-only-for-release + variables: + BUILD_DIR: build-package + CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None + CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF + CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF + +# Right now we need to use an ugly hack to move the original build directory +# and the ccache files out of the build archive, as it is not possible to use +# the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203 +# As soon as this is resolved we should use the separate build directories instead of +# manually hacking things. TODO: #3361 +.archive:build: + extends: + - .variables:default + - .gromacs:base:build + - .use-ccache + - .before_script:default + - .docs:build + image: gromacs/ci-docs-llvm:2020 + variables: + KUBERNETES_CPU_LIMIT: 4 + KUBERNETES_CPU_REQUEST: 2 + KUBERNETES_MEMORY_LIMIT: 4Gi + script: + - rm -rf gromacs*tar.gz + - cd $BUILD_DIR + - cmake --build . --target gmx -- -j4 2>&1 | tee buildLogFile.log + - cmake --build . --target man -- -j4 2>&1 | tee manBuildLogFile.log + - cmake --build . --target completion -- -j4 2>&1 | tee completionBuildLogFile.log + - cmake --build . --target install-guide -- -j4 2>&1 | tee installGuideBuildLogFile.log + - cmake --build . --target reference_checksum -- -j4 2>&1 | tee checksumBuildLogFile.log + - cmake --build . --target package_source -- -j4 2>&1 | tee packageBuildLogFile.log + - awk '/warning/,/warning.*generated|^$/' buildLogFile.log manBuildLogFile.log + completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log + packageBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true + - grep "cannot be built" buildLogFile.log manBuildLogFile.log + completionBuildLogFile.log installGuideBuildLogFile.log checksumBuildLogFile.log + packageBuildLogFile.log | tee -a buildErrors.log || true + - if [ -s buildErrors.log ] ; then echo "Found warnings during build"; cat buildErrors.log; exit 1; fi + - mv gromacs*tar.gz .. + - cd .. + # ugly hack begins here + - mkdir tempdir + - SOURCETARBALL=`ls gromacs*tar.gz` + - mv $SOURCETARBALL tempdir + - cd tempdir + - tar -xf $SOURCETARBALL + - rm -rf gromacs-*/$BUILD_DIR + - tar -c --gzip -f $SOURCETARBALL gromacs-*/ + - mv $SOURCETARBALL .. + - cd .. + - rm -rf tempdir + # Hack ends here + artifacts: + when: always + expire_in: 1 week + paths: + - $BUILD_DIR/*log + - gromacs*tar.gz + +archive:build: + extends: + - .archive:build + - .rules:nightly-not-for-release + stage: nightly-build + needs: + - job: archive:configure:nightly + variables: + BUILD_DIR: build-package + +archive:package: + extends: + - .archive:build + - .rules:nightly-only-for-release + stage: release-package + needs: + - job: archive:configure:release + variables: + BUILD_DIR: build-package + +release-verify: + image: biophysics/gcc-gromacs + stage: release-verify + extends: + - .variables:default + - .rules:nightly-only-for-release + cache: {} + + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + script: + - VERSION=`cat version.json | + python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` + - if [[ $GROMACS_RELEASE != "true" ]] ; then + VERSION=$VERSION-dev ; + fi + - REGTEST_COMPARE=`cat version.json | + python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"` + - SOURCENAME=gromacs-$VERSION + - SOURCETARBALL=$SOURCENAME.tar.gz + - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'` + - REGTESTNAME=regressiontests-$VERSION + - REGTESTTARBALL=$REGTESTNAME.tar.gz + - REGTEST_MD5SUM=`md5sum $REGTESTTARBALL | awk '{print $1}'` + - echo "$SOURCETARBALL md5sum = $SOURCE_MD5SUM" + - echo "$REGTESTTARBALL md5sum = $REGTEST_MD5SUM" + - echo "$REGTESTTARBALL reference md5sum = $REGTEST_COMPARE" + - if [[ $REGTEST_COMPARE != $REGTEST_MD5SUM && $GROMACS_RELEASE == "true" ]] ; then + echo "Mismatch in regressiontest md5sums"; + exit 1; + fi + dependencies: + - archive:package + - regressiontests:package + - prepare-release-version + +archive:nightly-webpage: + extends: + - .webpage:build + - .rules:nightly-not-for-release + stage: nightly-deploy + needs: + - job: linkchecker + artifacts: false + - job: webpage:build + variables: + BUILD_DIR: build-docs + script: + - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ + artifacts: + when: always + paths: + - webpage.tar.gz + +archive:full-release: + extends: + - .webpage:build + - .rules:nightly-only-for-release + stage: release-deploy + dependencies: + - webpage:build:release + - archive:package + - regressiontests:package + variables: + BUILD_DIR: release-doc-builds + script: + - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ + artifacts: + when: always + paths: + - webpage.tar.gz + - gromacs-*tar.gz + - regressiontests-*tar.gz + diff --git a/admin/ci-templates/.build-docs-template.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml similarity index 55% rename from admin/ci-templates/.build-docs-template.yml rename to admin/gitlab-ci/documentation.gitlab-ci.yml index 7312348354..7c105a7c57 100644 --- a/admin/ci-templates/.build-docs-template.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -1,14 +1,14 @@ -.configure-docs-template: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:2020 +# Build documentation and web page. + +.docs:configure: + extends: + - .variables:default + - .gromacs:base:configure + - .before_script:default + # TODO (#3480) this should be organized more like the current documentation.py script + image: gromacs/ci-docs-llvm:2020 stage: configure-build - only: - refs: - - web - - pushes - - schedules - - merge_requests + cache: {} variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -20,8 +20,6 @@ CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" - script: - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then REL_OPTION="-DGMX_BUILD_TARBALL=ON" ; @@ -38,25 +36,25 @@ - echo $CMAKE_GMXAPI_OPTIONS - echo $REL_OPTION - if [[ -d $BUILD_DIR ]] ; then - rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; + rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; else - echo "Preparing new build directory" ; - mkdir $BUILD_DIR ; + echo "Preparing new build directory" ; + mkdir $BUILD_DIR ; fi - cd $BUILD_DIR - cmake .. - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - $CMAKE_COMPILER_SCRIPT - $CMAKE_EXTRA_OPTIONS - $CMAKE_SIMD_OPTIONS - $CMAKE_MPI_OPTIONS - $CMAKE_PRECISION_OPTIONS - $CMAKE_BUILD_TYPE_OPTIONS - $CMAKE_GPU_OPTIONS - $CMAKE_GMXAPI_OPTIONS - $REL_OPTION - -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON - 2>&1 | tee cmakeLog.log + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + $CMAKE_COMPILER_SCRIPT + $CMAKE_EXTRA_OPTIONS + $CMAKE_SIMD_OPTIONS + $CMAKE_MPI_OPTIONS + $CMAKE_PRECISION_OPTIONS + $CMAKE_BUILD_TYPE_OPTIONS + $CMAKE_GPU_OPTIONS + $CMAKE_GMXAPI_OPTIONS + $REL_OPTION + -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON + 2>&1 | tee cmakeLog.log - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - cd .. @@ -67,72 +65,16 @@ paths: - $BUILD_DIR -.build-docs-binary-template: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:2020 - only: - refs: - - web - - pushes - - schedules - - merge_requests - script: - - cd $BUILD_DIR - - cmake --build . --target gmx -- -j8 - - cd .. - artifacts: - name: docs-artifacts-$CI_COMMIT_REF_SLUG - when: always - expire_in: 1 week - paths: - - $BUILD_DIR - -.build-docs-webpage-template: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:2020 - only: - refs: - - web - - pushes - - schedules - - merge_requests - variables: - KUBERNETES_CPU_LIMIT: 4 - KUBERNETES_CPU_REQUEST: 2 - KUBERNETES_MEMORY_LIMIT: 4Gi - script: - - cd $BUILD_DIR - - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log - | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true - - cmake --build . --target webpage -- -j4 - - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true - - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true - - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true - - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi - - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi - - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi - - cd .. - artifacts: - name: docs-artifacts-$CI_COMMIT_REF_SLUG - when: always - expire_in: 1 week - paths: - - $BUILD_DIR/*.log - - $BUILD_DIR/docs/html - - $BUILD_DIR/docs/manual/gromacs.log - - $BUILD_DIR/docs/manual/all-output.log - - $BUILD_DIR/docs/doxygen/doxygen-user.log - - $BUILD_DIR/docs/doxygen/doxygen-lib.log - - $BUILD_DIR/docs/doxygen/doxygen-full.log - - $BUILD_DIR/docs/sphinx-html.log - -.configure-docs-release-template: - # Built by admin/dockerfiles/ci-docs-clang - # TODO this should be organized more like the current documentation.py script - image: gromacs/ci-docs-clang:2020 +.docs:configure:release: + extends: + - .variables:default + - .gromacs:base:configure + - .before_script:default + - .rules:nightly-only-for-release + # TODO (#3480) this should be organized more like the current documentation.py script + image: gromacs/ci-docs-llvm:2020 stage: release-configure + cache: {} variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -144,18 +86,9 @@ CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" - CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" RELEASE_BUILD_DIR: release-doc-builds RELEASE_SOURCE: release-source-from-tarball RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball - - only: - refs: - - web - - schedules - - triggers - variables: - - $GROMACS_RELEASE script: - if [[ $GROMACS_RELEASE == "true" ]] ; then REL_OPTION="-DGMX_BUILD_TARBALL=ON" ; @@ -172,12 +105,12 @@ - echo $CMAKE_GMXAPI_OPTIONS - echo $REL_OPTION - VERSION=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` + python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` - if [[ $GROMACS_RELEASE != "true" ]] ; then - VERSION=$VERSION-dev ; + VERSION=$VERSION-dev ; fi - REGTEST_COMPARE=`cat version.json | - python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"` + python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"` - SOURCENAME=gromacs-$VERSION - SOURCETARBALL=$SOURCENAME.tar.gz - REGTESTNAME=regressiontests-$VERSION @@ -188,8 +121,8 @@ - mv $SOURCETARBALL $RELEASE_TARBALLS - mv $REGTESTTARBALL $RELEASE_TARBALLS - cd $RELEASE_TARBALLS -# We rename the source and regressiontest directories -# to have names for them that don't change for different versions. + # We rename the source and regressiontest directories + # to have names for them that don't change for different versions. - tar -xf $SOURCETARBALL - mv $SOURCENAME ../$RELEASE_SOURCE - tar -xf $REGTESTTARBALL @@ -205,28 +138,28 @@ - echo $CMAKE_BUILD_TYPE_OPTIONS - echo $CMAKE_GMXAPI_OPTIONS - if [[ -d $RELEASE_BUILD_DIR ]] ; then - echo "Cleaning up build directory" ; - rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ; + echo "Cleaning up build directory" ; + rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ; else - echo "Preparing new build directory" ; - mkdir $RELEASE_BUILD_DIR ; + echo "Preparing new build directory" ; + mkdir $RELEASE_BUILD_DIR ; fi - cd $RELEASE_BUILD_DIR - cmake ../$RELEASE_SOURCE/ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - $CMAKE_COMPILER_SCRIPT - $CMAKE_EXTRA_OPTIONS - $CMAKE_SIMD_OPTIONS - $CMAKE_MPI_OPTIONS - $CMAKE_PRECISION_OPTIONS - $CMAKE_BUILD_TYPE_OPTIONS - $CMAKE_GPU_OPTIONS - $CMAKE_GMXAPI_OPTIONS - $REL_OPTION - "-DSOURCE_MD5SUM=$SOURCE_MD5SUM" - "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS" - -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON - 2>&1 | tee cmakeLog.log + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + $CMAKE_COMPILER_SCRIPT + $CMAKE_EXTRA_OPTIONS + $CMAKE_SIMD_OPTIONS + $CMAKE_MPI_OPTIONS + $CMAKE_PRECISION_OPTIONS + $CMAKE_BUILD_TYPE_OPTIONS + $CMAKE_GPU_OPTIONS + $CMAKE_GMXAPI_OPTIONS + $REL_OPTION + "-DSOURCE_MD5SUM=$SOURCE_MD5SUM" + "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS" + -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON + 2>&1 | tee cmakeLog.log - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - cd .. @@ -238,3 +171,138 @@ - $RELEASE_BUILD_DIR - $RELEASE_REGRESSIONTESTS - $RELEASE_SOURCE + +docs:configure: + extends: + - .docs:configure + - .rules:basic-push + tags: [] + variables: + BUILD_DIR: build-docs + CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on + +.docs:build: + # TODO (#3480) this should be organized more like the current documentation.py script + image: gromacs/ci-docs-llvm:2020 + script: + - cd $BUILD_DIR + - cmake --build . --target gmx -- -j8 + - cd .. + artifacts: + name: docs-artifacts-$CI_COMMIT_REF_SLUG + when: always + expire_in: 1 week + paths: + - $BUILD_DIR + +docs:build: + extends: + - .variables:default + - .gromacs:base:build + - .use-ccache + - .docs:build + - .before_script:default + - .rules:basic-push + stage: documentation + tags: [] + needs: + - job: docs:configure + variables: + BUILD_DIR: build-docs + +# The manual build is built separately so that errors in converting +# Sphinx to LaTeX and compiling can always be found in the +# all-output.txt file, while avoiding many thousands of lines of spam +# from pdflatex for normal builds. This does reduce the available +# parallel utilization, and so increases the build time. +# +# TODO why are the doxygen and sphinx log files disappearing +# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job +.webpage:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + # TODO (#3480) this should be organized more like the current documentation.py script + image: gromacs/ci-docs-llvm:2020 + cache: {} + variables: + KUBERNETES_CPU_LIMIT: 4 + KUBERNETES_CPU_REQUEST: 2 + KUBERNETES_MEMORY_LIMIT: 4Gi + script: + - cd $BUILD_DIR + - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log + | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true + - cmake --build . --target webpage -- -j4 + - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true + - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true + - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true + - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi + - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi + - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi + - cd .. + artifacts: + name: docs-artifacts-$CI_COMMIT_REF_SLUG + when: always + expire_in: 1 week + paths: + - $BUILD_DIR/*.log + - $BUILD_DIR/docs/html + - $BUILD_DIR/docs/manual/gromacs.log + - $BUILD_DIR/docs/manual/all-output.log + - $BUILD_DIR/docs/doxygen/doxygen-user.log + - $BUILD_DIR/docs/doxygen/doxygen-lib.log + - $BUILD_DIR/docs/doxygen/doxygen-full.log + - $BUILD_DIR/docs/sphinx-html.log + +webpage:build: + extends: + - .webpage:build + - .rules:basic-push + stage: post-test + tags: [] + needs: + - job: docs:build + - job: check-source + artifacts: false + variables: + BUILD_DIR: build-docs + +webpage:configure: + extends: + - .docs:configure:release + - .rules:nightly-only-for-release + cache: {} + dependencies: + - archive:package + - regressiontests:package + - prepare-release-version + variables: + CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on + + +webpage:dependencies: + extends: + - .gromacs:base:build + - .variables:default + - .use-ccache + - .docs:build + - .before_script:default + - .rules:nightly-only-for-release + stage: release-build + needs: + - job: webpage:configure + variables: + BUILD_DIR: release-doc-builds + +webpage:build:release: + extends: + - .webpage:build + - .rules:nightly-only-for-release + dependencies: + - webpage:dependencies + - webpage:configure + stage: release-prepare-deploy + variables: + BUILD_DIR: release-doc-builds diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml new file mode 100644 index 0000000000..5734819560 --- /dev/null +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -0,0 +1,129 @@ +# Mix-in job definitions. + +# Centralized definitions of common job parameter values. + +.variables:default: + variables: + KUBERNETES_CPU_LIMIT: 8 + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_LIMIT: 8Gi + KUBERNETES_EXTENDED_RESOURCE_NAME: "" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 + CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-release-2020" + BUILD_DIR: build + INSTALL_DIR: install + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" +# TODO: Get these from updated Docker images. + +# Our generic before_script to install dependencies and prepare the ccache directory. +.before_script:default: + before_script: + - mkdir -p ccache + - export CCACHE_BASEDIR=${PWD} + - export CCACHE_DIR=${PWD}/ccache + +# Jobs that run for new commits and pipelines triggered by schedules or +# through the web interface, unless GROMACS_RELEASE is set. Excluded from +# extra pipelines generated by merge request events. +# Includes non-gromacs projects. Note that jobs using this rule are +# eligible to run on non-gromacs project infrastructure, and should therefore +# override the default *tag* parameter to exclude tags specific to the GROMACS +# GitLab Runner infrastructure. I.e. in the job definition, set `tags: []` +.rules:basic-push: + rules: + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: always + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + +# Jobs that run for merge requests and schedules, but not when GROMACS_RELEASE +# is set. Excludes non-gromacs projects. +.rules:merge-requests: + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + +# Rule to run a job only in nightly release-preparation pipelines. +# Checks if the GROMACS_RELEASE variable was set (typically through the GitLab web interface). +# Excludes merge_requests and non-gromacs projects. +# TODO: Update to *rules* syntax. +.rules:nightly-only-for-release: + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE && $CI_PIPELINE_SOURCE == "web"' + when: always + - if: '$GROMACS_RELEASE && $CI_PIPELINE_SOURCE == "schedule"' + when: always + +# Jobs that run on schedules, but not for merge requests or when GROMACS_RELEASE +# is set. Excludes non-gromacs projects. +.rules:nightly-not-for-release: + rules: + - if: '$CI_PROJECT_NAMESPACE != "gromacs"' + when: never + - if: '$GROMACS_RELEASE' + when: never + - if: '$CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule"' + when: always + +# Behavioral templates + +# Use a persistent compiler cache to speed up rebuilds for a single job. +.use-ccache: + cache: + key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - ccache/ + +# Tool chains + +.use-cuda: + variables: + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" + CMAKE_GPU_OPTIONS: -DGMX_GPU=ON -DGMX_USE_CUDA=ON + +.use-mpi: + variables: + CMAKE_MPI_OPTIONS: "-DGMX_MPI=ON" + +.use-opencl: + variables: + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" + CMAKE_GPU_OPTIONS: -DGMX_GPU=ON -DGMX_USE_OPENCL=ON + +# Base definition for using gcc. +.use-gcc:base: + variables: + CMAKE_COMPILER_SCRIPT: -DCMAKE_C_COMPILER=gcc-$COMPILER_MAJOR_VERSION -DCMAKE_CXX_COMPILER=g++-$COMPILER_MAJOR_VERSION + before_script: + - mkdir -p ccache + - export CCACHE_BASEDIR=${PWD} + - export CCACHE_DIR=${PWD}/ccache + +# Base definition for using clang. +.use-clang:base: + variables: + CMAKE_COMPILER_SCRIPT: -DCMAKE_C_COMPILER=clang-$COMPILER_MAJOR_VERSION -DCMAKE_CXX_COMPILER=clang++-$COMPILER_MAJOR_VERSION + before_script: + - mkdir -p ccache + - export CCACHE_BASEDIR=${PWD} + - export CCACHE_DIR=${PWD}/ccache + - export ASAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer + diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml new file mode 100644 index 0000000000..1285c582c6 --- /dev/null +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -0,0 +1,813 @@ +# Build, test, and install variously configured GROMACS installations. + +# Initial build that always needs to pass before follow-up stages run +simple-build: + extends: + - .variables:default + - .use-ccache + - .use-gcc:base + - .rules:basic-push + stage: pre-build + image: gromacs/cmake-3.9.6-gcc-5-cuda-9.0-openmpi:2020 + tags: [] + variables: + KUBERNETES_CPU_LIMIT: 8 + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_LIMIT: 8Gi + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" + CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" + CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" + CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=OFF" + COMPILER_MAJOR_VERSION: 5 + BUILD_DIR: simple-build + script: + - echo $CMAKE_COMPILER_SCRIPT + - echo $CMAKE_EXTRA_OPTIONS + - echo $CMAKE_SIMD_OPTIONS + - echo $CMAKE_GPU_OPTIONS + - echo $CMAKE_MPI_OPTIONS + - echo $CMAKE_PRECISION_OPTIONS + - echo $CMAKE_BUILD_TYPE_OPTIONS + - echo $CMAKE_GMXAPI_OPTIONS + - if [[ -d $BUILD_DIR ]] ; then + echo "Cleaning up build directory" ; + rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; + else + echo "Preparing new build directory" ; + mkdir $BUILD_DIR ; + fi + - cd $BUILD_DIR + - cmake .. + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + $CMAKE_COMPILER_SCRIPT + $CMAKE_EXTRA_OPTIONS + $CMAKE_SIMD_OPTIONS + $CMAKE_MPI_OPTIONS + $CMAKE_PRECISION_OPTIONS + $CMAKE_BUILD_TYPE_OPTIONS + $CMAKE_GPU_OPTIONS + $CMAKE_GMXAPI_OPTIONS + -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON + 2>&1 | tee cmakeLog.log + - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log + - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi + - cmake --build . -- -j8 2>&1 | tee buildLogFile.log + - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log + - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log + | grep -v "CMake" | tee buildErrors.log || true + - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true + - cmake --build . --target install 2>&1 | tee installBuildLogFile.log + - if [ -s buildErrors.log ] ; then echo "Found compiler warning during build"; cat buildErrors.log; exit 1; fi + - ctest -D ExperimentalTest --output-on-failure | tee ctestLog.log || true + - awk '/The following tests FAILED/,/^Errors while running CTest|^$/' + ctestLog.log | tee ctestErrors.log + - xsltproc scripts/CTest2JUnit.xsl Testing/`head -n 1 < Testing/TAG`/Test.xml > JUnitTestResults.xml + - if [ -s ctestErrors.log ] ; then + echo "Error during running ctest"; + exit 1; + fi + - cd .. + artifacts: + reports: + junit: $BUILD_DIR/JUnitTestResults.xml + paths: + - $BUILD_DIR/*log + when: always + expire_in: 1 week + +# Jobs running in first pre-build stage + +.gromacs:base:configure: + extends: + - .variables:default + stage: configure-build + cache: {} + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + CMAKE_COMPILER_SCRIPT: "" + CMAKE_EXTRA_OPTIONS: "" + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" + CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug" + CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" + script: + - echo $CMAKE_COMPILER_SCRIPT + - echo $CMAKE_EXTRA_OPTIONS + - echo $CMAKE_SIMD_OPTIONS + - echo $CMAKE_GPU_OPTIONS + - echo $CMAKE_MPI_OPTIONS + - echo $CMAKE_PRECISION_OPTIONS + - echo $CMAKE_BUILD_TYPE_OPTIONS + - echo $CMAKE_GMXAPI_OPTIONS + - if [[ -d $BUILD_DIR ]] ; then + rm -rf $BUILD_DIR && mkdir $BUILD_DIR ; + else + echo "Preparing new build directory" ; + mkdir $BUILD_DIR ; + fi + - cd $BUILD_DIR + - cmake .. + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + $CMAKE_COMPILER_SCRIPT + $CMAKE_EXTRA_OPTIONS + $CMAKE_SIMD_OPTIONS + $CMAKE_MPI_OPTIONS + $CMAKE_PRECISION_OPTIONS + $CMAKE_BUILD_TYPE_OPTIONS + $CMAKE_GPU_OPTIONS + $CMAKE_GMXAPI_OPTIONS + -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON + 2>&1 | tee cmakeLog.log + - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log + - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi + - cd .. + artifacts: + when: always + paths: + - $BUILD_DIR + +.gromacs:base:release:configure: + extends: + - .variables:default + stage: release-configure + cache: {} + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + CMAKE_COMPILER_SCRIPT: "" + CMAKE_EXTRA_OPTIONS: "" + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" + CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithAssert" + CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF" + CMAKE_REGRESSIONTEST_OPTIONS: "-DREGRESSIONTEST_PATH=../\\$RELEASE_REGRESSIONTESTS" + RELEASE_BUILD_DIR: release-builds + RELEASE_SOURCE: release-source-from-tarball + RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball + script: + - VERSION=`cat version.json | + python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` + - if [[ $GROMACS_RELEASE != "true" ]] ; then + VERSION=$VERSION-dev ; + fi + - SOURCENAME=gromacs-$VERSION + - SOURCETARBALL=$SOURCENAME.tar.gz + - REGTESTNAME=regressiontests-$VERSION + - REGTESTTARBALL=$REGTESTNAME.tar.gz + - echo "$SOURCETARBALL" + - echo "$REGTESTTARBALL" + - RELEASE_TARBALLS=release-tarballs + - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS + - mv $SOURCETARBALL $RELEASE_TARBALLS + - mv $REGTESTTARBALL $RELEASE_TARBALLS + - cd $RELEASE_TARBALLS + # We rename the source and regressiontest directories + # to have names for them that don't change for different versions. + - tar -xf $SOURCETARBALL + - mv $SOURCENAME ../$RELEASE_SOURCE + - tar -xf $REGTESTTARBALL + - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS + - rm $SOURCETARBALL $REGTESTTARBALL + - cd .. + - echo $CMAKE_COMPILER_SCRIPT + - echo $CMAKE_EXTRA_OPTIONS + - echo $CMAKE_SIMD_OPTIONS + - echo $CMAKE_GPU_OPTIONS + - echo $CMAKE_MPI_OPTIONS + - echo $CMAKE_PRECISION_OPTIONS + - echo $CMAKE_BUILD_TYPE_OPTIONS + - echo $CMAKE_GMXAPI_OPTIONS + - if [[ -d $RELEASE_BUILD_DIR ]] ; then + echo "Cleaning up build directory" ; + rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ; + else + echo "Preparing new build directory" ; + mkdir $RELEASE_BUILD_DIR ; + fi + - cd $RELEASE_BUILD_DIR + - cmake ../$RELEASE_SOURCE/ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + $CMAKE_COMPILER_SCRIPT + $CMAKE_EXTRA_OPTIONS + $CMAKE_SIMD_OPTIONS + $CMAKE_MPI_OPTIONS + $CMAKE_PRECISION_OPTIONS + $CMAKE_BUILD_TYPE_OPTIONS + $CMAKE_GPU_OPTIONS + $CMAKE_GMXAPI_OPTIONS + $CMAKE_REGRESSIONTEST_OPTIONS + -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR + 2>&1 | tee cmakeLog.log + - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log + - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi + - cd .. + artifacts: + when: always + paths: + - $RELEASE_BUILD_DIR + - $RELEASE_REGRESSIONTESTS + - $RELEASE_SOURCE + +# Templates for configuration stage + +gromacs:gcc-7:configure: + extends: + - .gromacs:base:configure + - .use-gcc:base + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AVX2_256" + CMAKE_GPU_OPTIONS: "-DGMX_GPU=ON -DGMX_USE_OPENCL=ON" + COMPILER_MAJOR_VERSION: 7 + +gromacs:gcc-6:configure: + extends: + - .gromacs:base:configure + - .use-gcc:base + - .use-opencl + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + variables: + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" + COMPILER_MAJOR_VERSION: 6 + CMAKE_EXTRA_OPTIONS: "-DGMX_EXTERNAL_CLFFT=ON" + +gromacs:clang-TSAN:configure: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + variables: + COMPILER_MAJOR_VERSION: 8 + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=TSAN" + +gromacs:clang-8:configure: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: 8 + +gromacs:clang-3.6:configure: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: "3.6" + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=ON" + CMAKE_MPI_OPTIONS: "-DGMX_OPENMP=OFF" + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" + +gromacs:clang-static-analyzer:configure: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + variables: + CMAKE_COMPILER_SCRIPT: "-DCMAKE_CXX_COMPILER=/usr/local/libexec/c++-analyzer -DCMAKE_C_COMPILER=gcc" + CMAKE_EXTRA_OPTIONS: "-DGMX_CLANG_ANALYZER=ON -DGMX_OPENMP=OFF -DGMX_USE_RDTSCP=OFF -DGMX_FFT_LIBRARY=fftpack -DGMX_DEVELOPER_BUILD=ON" + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" + +gromacs:clang-ASAN:configure: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + variables: + COMPILER_MAJOR_VERSION: 8 + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=ASAN" + +gromacs:gcc-6:release:configure: + extends: + - .gromacs:base:release:configure + - .use-gcc:base + - .use-cuda + - .rules:nightly-only-for-release + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: 6 + RELEASE_BUILD_DIR: release-builds-gcc + CMAKE_EXTRA_OPTIONS: "-DGMX_BUILD_MDRUN_ONLY=ON" + CMAKE_BUILD_TYPE_OPTIONS : "-DCMAKE_BUILD_TYPE=RelWithDebug" + CMAKE_REGRESSIONTEST_OPTIONS: "" + dependencies: + - archive:package + - regressiontests:package + - prepare-release-version + +gromacs:gcc-7:release:configure: + extends: + - .gromacs:base:release:configure + - .use-gcc:base + - .rules:nightly-only-for-release + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: 7 + RELEASE_BUILD_DIR: release-builds-gcc + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithAssert" + dependencies: + - archive:package + - regressiontests:package + - prepare-release-version + +gromacs:clang-3.6:release:configure: + extends: + - .gromacs:base:release:configure + - .use-clang:base + - .rules:nightly-only-for-release + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: "3.6" + RELEASE_BUILD_DIR: release-builds-clang + CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=ON" + CMAKE_MPI_OPTIONS: "-DGMX_OPENMP=OFF" + CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=SSE4.1" + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithAssert" + dependencies: + - archive:package + - regressiontests:package + - prepare-release-version + +gromacs:clang-8:release:configure: + extends: + - .gromacs:base:release:configure + - .use-clang:base + - .use-cuda + - .rules:nightly-only-for-release + image: gromacs/cmake-3.15.7-llvm-8-cuda-10.1-openmpi:2020 + variables: + COMPILER_MAJOR_VERSION: 8 + RELEASE_BUILD_DIR: release-builds-clang + CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=RelWithDebug" + +# Jobs running during build stage + +.gromacs:base:build: + stage: build + script: + - cd $BUILD_DIR + - cmake --build . -- -j8 2>&1 | tee buildLogFile.log + - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log + - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log + | grep -v "CMake" | tee buildErrors.log || true + - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true + - cmake --build . --target install 2>&1 | tee installBuildLogFile.log + - if [ -s buildErrors.log ] ; then echo "Found compiler warning during build"; cat buildErrors.log; exit 1; fi + - cd .. + artifacts: + when: always + paths: + - $BUILD_DIR + - $INSTALL_DIR/ + +.gromacs:static-analyzer-build: + stage: build + script: + - cd $BUILD_DIR + - scan-build -o scan_html cmake --build . -- -j8 2>&1 | tee buildLogFile.log + - awk '/warning/,/warning.*generated|^$/' buildLogFile.log + | grep -v "CMake" | tee buildErrors.log || true + - grep "cannot be built" buildLogFile.log | tee -a buildErrors.log || true + - if [ -s buildErrors.log ] ; then echo "Found compiler warning during build"; cat buildErrors.log; exit 1; fi + - cd .. + artifacts: + when: always + paths: + - $BUILD_DIR/scan_html + +gromacs:gcc-7:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + needs: + - job: gromacs:gcc-7:configure + artifacts: true + +gromacs:gcc-6:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + needs: + - job: gromacs:gcc-6:configure + +gromacs:clang-TSAN:build: + extends: + - .variables:default + - .gromacs:base:build + - .use-clang:base + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-TSAN:configure + +gromacs:clang-ASAN:build: + extends: + - .variables:default + - .gromacs:base:build + - .use-clang:base + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-ASAN:configure + +gromacs:clang-static-analyzer:build: + extends: + - .variables:default + - .gromacs:static-analyzer-build + - .use-clang:base + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-static-analyzer:configure + +gromacs:clang-8:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + needs: + - job: gromacs:clang-8:configure + +gromacs:clang-3.6:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:merge-requests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + needs: + - job: gromacs:clang-3.6:configure + +gromacs:gcc-6:release:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:nightly-only-for-release + stage: release-build + variables: + BUILD_DIR: release-builds-gcc + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + needs: + - job: gromacs:gcc-6:release:configure + +gromacs:gcc-7:release:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:nightly-only-for-release + stage: release-build + variables: + BUILD_DIR: release-builds-gcc + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + needs: + - job: gromacs:gcc-7:release:configure + +gromacs:clang-3.6:release:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:nightly-only-for-release + stage: release-build + variables: + BUILD_DIR: release-builds-clang + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + needs: + - job: gromacs:clang-3.6:release:configure + +gromacs:clang-8:release:build: + extends: + - .variables:default + - .gromacs:base:build + - .before_script:default + - .use-ccache + - .rules:nightly-only-for-release + stage: release-build + variables: + BUILD_DIR: release-builds-clang + image: gromacs/cmake-3.15.7-llvm-8-cuda-10.1-openmpi:2020 + needs: + - job: gromacs:clang-8:release:configure + +# Jobs running during test stage + +.gromacs:base:test: + extends: + - .variables:default + stage: test + cache: {} + retry: + max: 1 + variables: + CTEST_RUN_MODE: "ExperimentalTest" + script: + - cd $BUILD_DIR + # Needed to run MPI enabled code in the docker images, until we set up different users + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + - ctest -D $CTEST_RUN_MODE --output-on-failure | tee ctestLog.log || true + - awk '/The following tests FAILED/,/^Errors while running CTest|^$/' + ctestLog.log | tee ctestErrors.log + - if [ $CTEST_RUN_MODE == "ExperimentalTest" ] ; then + xsltproc scripts/CTest2JUnit.xsl Testing/`head -n 1 < Testing/TAG`/Test.xml > JUnitTestResults.xml; + else + touch JUnitTestResults.xml; + fi + - if [ -s ctestErrors.log ] ; then + echo "Error during running ctest"; + exit 1; + fi + - cd .. + artifacts: + reports: + junit: $BUILD_DIR/JUnitTestResults.xml + paths: + - $BUILD_DIR/Testing + - $BUILD_DIR/tests + when: always + expire_in: 1 week + +.gromacs:base:regressiontest: + extends: + - .variables:default + variables: + REGRESSIONTEST_TOTAL_RANK_NUMBER: 4 + REGRESSIONTEST_OMP_RANK_NUMBER: 2 + REGRESSIONTEST_PME_RANK_NUMBER: 1 + REGRESSIONTEST_MPI_RUN_COMMAND: "" + REGRESSIONTEST_DOUBLE: "" + REGRESSIONTEST_PARALLEL: "-nt" + stage: test + cache: {} + retry: + max: 1 + script: + # This should go away once we are able to run ASAN on mdrun + - export ASAN_OPTIONS="detect_leaks=0" + # Needed to run MPI enabled code in the docker images, until we set up different users + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + - tar xzf gmx-regressiontests.tar.gz + - source $INSTALL_DIR/bin/GMXRC + - cd regressiontests + - perl gmxtest.pl $REGRESSIONTEST_PARALLEL $REGRESSIONTEST_TOTAL_RANK_NUMBER -ntomp $REGRESSIONTEST_OMP_RANK_NUMBER -npme $REGRESSIONTEST_PME_RANK_NUMBER $REGRESSIONTEST_DOUBLE $REGRESSIONTEST_MPI_RUN_COMMAND -xml all + artifacts: + reports: + junit: regressiontests/gmxtest.xml + when: always + expire_in: 1 week + +gromacs:gcc-7:test: + extends: + - .gromacs:base:test + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + KUBERNETES_EXTENDED_RESOURCE_NAME: "amd.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 + needs: + - job: gromacs:gcc-7:build + +gromacs:gcc-6:test: + extends: + - .gromacs:base:test + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + variables: + KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 + needs: + - job: gromacs:gcc-6:build + +gromacs:clang-8:test: + extends: + - .gromacs:base:test + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + needs: + - job: gromacs:clang-8:build + +gromacs:clang-TSAN:test: + extends: + - .gromacs:base:test + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-TSAN:build + +gromacs:clang-ASAN:test: + extends: + - .gromacs:base:test + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + variables: + CTEST_RUN_MODE: "ExperimentalMemCheck" + needs: + - job: gromacs:clang-ASAN:build + +gromacs:clang-3.6:test: + extends: + - .gromacs:base:test + - .rules:merge-requests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + needs: + - job: gromacs:clang-3.6:build + +gromacs:gcc-7:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + KUBERNETES_EXTENDED_RESOURCE_NAME: "amd.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 + REGRESSIONTEST_PME_RANK_NUMBER: 0 + REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 + REGRESSIONTEST_OMP_RANK_NUMBER: 1 + needs: + - job: gromacs:gcc-7:build + - job: regressiontests:prepare + +gromacs:gcc-6:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:merge-requests + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + variables: + KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 + REGRESSIONTEST_PME_RANK_NUMBER: 0 + REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 + REGRESSIONTEST_OMP_RANK_NUMBER: 1 + needs: + - job: gromacs:gcc-6:build + - job: regressiontests:prepare + +gromacs:clang-8:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + needs: + - job: gromacs:clang-8:build + - job: regressiontests:prepare + +gromacs:clang-TSAN:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-TSAN:build + - job: regressiontests:prepare + +gromacs:clang-ASAN:regressiontest: + extends: + - .gromacs:base:regressiontest + - .use-clang:base + - .rules:merge-requests + image: gromacs/cmake-3.15.7-llvm-8-tsan:2020 + needs: + - job: gromacs:clang-ASAN:build + - job: regressiontests:prepare + +gromacs:clang-3.6:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:merge-requests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + variables: + REGRESSIONTEST_DOUBLE: "-double" + REGRESSIONTEST_OMP_RANK_NUMBER: 0 + needs: + - job: gromacs:clang-3.6:build + - job: regressiontests:prepare + +gromacs:gcc-6:release:test: + extends: + - .gromacs:base:test + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.9.6-gcc-6-cuda-10.1-nvidiaopencl-clfft-openmpi:2020 + variables: + BUILD_DIR: release-builds-gcc + needs: + - job: gromacs:gcc-6:release:configure + - job: gromacs:gcc-6:release:build + +gromacs:gcc-7:release:test: + extends: + - .gromacs:base:test + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + BUILD_DIR: release-builds-gcc + needs: + - job: gromacs:gcc-7:release:configure + - job: gromacs:gcc-7:release:build + +gromacs:clang-3.6:release:test: + extends: + - .gromacs:base:test + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + variables: + BUILD_DIR: release-builds-clang + needs: + - job: gromacs:clang-3.6:release:configure + - job: gromacs:clang-3.6:release:build + +gromacs:clang-8:release:test: + extends: + - .gromacs:base:test + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.15.7-llvm-8-cuda-10.1-openmpi:2020 + variables: + BUILD_DIR: release-builds-clang + needs: + - job: gromacs:clang-8:release:configure + - job: gromacs:clang-8:release:build + +gromacs:gcc-7:release:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.9.6-gcc-7-amdopencl-clfft-openmpi:2020 + variables: + BUILD_DIR: release-builds-gcc + needs: + - job: gromacs:gcc-7:release:build + - job: regressiontests:package + +gromacs:clang-3.6:release:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.9.6-llvm-3.6-amdopencl-openmpi:2020 + variables: + BUILD_DIR: release-builds-clang + REGRESSIONTEST_DOUBLE: "-double" + REGRESSIONTEST_OMP_RANK_NUMBER: 0 + needs: + - job: gromacs:clang-3.6:release:build + - job: regressiontests:package + +gromacs:clang-8:release:regressiontest: + extends: + - .gromacs:base:regressiontest + - .rules:nightly-only-for-release + stage: release-tests + image: gromacs/cmake-3.15.7-llvm-8-cuda-10.1-openmpi:2020 + variables: + BUILD_DIR: release-builds-clang + KUBERNETES_EXTENDED_RESOURCE_NAME: "nvidia.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 1 + REGRESSIONTEST_PME_RANK_NUMBER: 0 + REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 + REGRESSIONTEST_OMP_RANK_NUMBER: 1 + + needs: + - job: gromacs:clang-8:release:build + - job: regressiontests:package + diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml new file mode 100644 index 0000000000..5eb97d21e6 --- /dev/null +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -0,0 +1,173 @@ +# Repository cleanliness. Source tidiness, linting, and policy compliance. + +clang-tidy:configure-push: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:basic-push + image: gromacs/cmake-3.11.4-llvm-8-openmpi:2020 + tags: [] + variables: + COMPILER_MAJOR_VERSION: 8 + BUILD_DIR: build-clang-tidy + CMAKE_EXTRA_OPTIONS: -DCLANG_TIDY=clang-tidy-$COMPILER_MAJOR_VERSION -DGMX_CLANG_TIDY=ON -DGMX_COMPILER_WARNINGS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + +clang-tidy:configure-schedule: + extends: + - .gromacs:base:configure + - .use-clang:base + - .rules:nightly-not-for-release + image: gromacs/cmake-3.11.4-llvm-8-openmpi:2020 + tags: [] + variables: + COMPILER_MAJOR_VERSION: 8 + BUILD_DIR: build-clang-tidy + CMAKE_EXTRA_OPTIONS: -DCLANG_TIDY=clang-tidy-$COMPILER_MAJOR_VERSION -DGMX_CLANG_TIDY=ON -DGMX_COMPILER_WARNINGS=ON + +clang-tidy:build: + extends: + - .gromacs:base:build + - .use-ccache + - .variables:default + - .rules:nightly-not-for-release + stage: source-check + image: gromacs/cmake-3.11.4-llvm-8-openmpi:2020 + needs: + - job: clang-tidy:configure-schedule + variables: + BUILD_DIR: build-clang-tidy + +clang-tidy:test: + extends: + - .gromacs:base:build + - .variables:default + - .rules:basic-push + stage: source-check + image: gromacs/cmake-3.11.4-llvm-8-openmpi:2020 + tags: [] + needs: + - job: clang-tidy:configure-push + variables: + COMPILER_MAJOR_VERSION: 8 + BUILD_DIR: build-clang-tidy + EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION + KUBERNETES_CPU_LIMIT: 4 + KUBERNETES_CPU_REQUEST: 2 + KUBERNETES_MEMORY_LIMIT: 4Gi + script: + # TODO (issue #3272) `master` is not appropriate for use on release-xxxx branches, how should we handle that? + # See discussion at https://gitlab.com/gromacs/gromacs/-/merge_requests/67 + - REV=$(git fetch -q https://gitlab.com/gromacs/gromacs.git release-2020 && git show -s --pretty=format:"%h" `git merge-base --fork-point FETCH_HEAD HEAD`) + - RUN_CLANG_TIDY=run-clang-tidy-$COMPILER_MAJOR_VERSION bash admin/clang-tidy.sh check --parallel=$KUBERNETES_CPU_LIMIT --warnings=clang-tidy.log --rev=$REV -B=$BUILD_DIR + - grep -iq "found code issues" clang-tidy.log | tee code-lint.txt || true + - if [ -s code-lint.txt ] ; then echo "clang-tidy.sh found issues"; exit 1; fi + artifacts: + when: on_failure + paths: + - clang-tidy.log + - code-lint.txt + +clang-format: + extends: + - .variables:default + - .rules:basic-push + cache: {} + stage: pre-build + image: gromacs/ci-docs-llvm:2020 + tags: [] + variables: + COMPILER_MAJOR_VERSION: 7 + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + EXTRA_INSTALLS: clang-format-$COMPILER_MAJOR_VERSION + script: + - export CLANG_FORMAT=clang-format-$COMPILER_MAJOR_VERSION + - admin/clang-format.sh check --rev=HEAD^ --warnings=clang-format.log + - grep -iq "needs formatting" clang-format.log | tee formatting.txt || true + - if [ -s formatting.txt ] ; then echo "clang-format.sh found issues"; exit 1; fi + artifacts: + when: on_failure + paths: + - clang-format.log + - formatting.txt + +copyright-check: + extends: + - .variables:default + - .rules:basic-push + cache: {} + stage: pre-build + image: gromacs/ci-docs-llvm:2020 + tags: [] + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + script: + - admin/copyright.sh check --rev=HEAD^ --warnings=copyright.log + - grep -iq "copyright year" copyright.log | tee years.log || true + - grep -iq "copyright header" copyright.log | tee headers.log || true + - if [[ -s years.log || -s headers.log ]] ; then + echo "Copyright information needs updating" ; + exit 1 ; + fi + artifacts: + when: on_failure + paths: + - copyright.log + - years.log + - headers.log + +check-source: + extends: + - .variables:default + - .docs:build + - .before_script:default + - .rules:basic-push + cache: {} + stage: source-check + tags: [] + needs: + - job: docs:build + artifacts: true + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + BUILD_DIR: build-docs + script: + - cd $BUILD_DIR + - cmake --build . --target check-source + - awk '/warning.*include style.*order/,/You can use.*rst|^$/' docs/doxygen/check-source.log | tee doxygenError.txt || true + - awk '/Traceback/,/.*rror|^$/' docs/doxygen/doxygen*log docs/doxygen/check-source.log | tee -a doxygenError.txt || true + - awk '/warning:/,/in doxygen.*|^$/' docs/doxygen/doxygen*log | tee -a doxygenError.txt || true + - if [ -s doxygenError.txt ] ; then echo "Found errors while running doxygen"; exit 1; fi + artifacts: + name: docs-artifacts-$CI_COMMIT_REF_SLUG + when: always + expire_in: 1 week + paths: + - $BUILD_DIR/docs/doxygen/doxygen-xml.log + - $BUILD_DIR/docs/doxygen/check-source.log + +linkchecker: + extends: + - .webpage:build + - .rules:nightly-not-for-release + stage: nightly-build + dependencies: + - webpage:build + variables: + KUBERNETES_CPU_LIMIT: 1 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_LIMIT: 2Gi + BUILD_DIR: build-docs + script: + - cd $BUILD_DIR + - linkchecker docs/html/index.html -f $CI_PROJECT_DIR/docs/linkcheckerrc -Fxml --ignore-url html-full + --ignore-url html-user --ignore-url html-lib --ignore-url .tar.gz --ignore-url _sources + -o xml + artifacts: + reports: + junit: $BUILD_DIR/linkchecker-out.xml diff --git a/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml new file mode 100644 index 0000000000..4d51fe5935 --- /dev/null +++ b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml @@ -0,0 +1,108 @@ +# +# Jobs to test gmxapi client (Python) packages +# + +# Base job definition for gmxapi 0.1 tests against GROMACS 2020 in clang-8 environments. +.gmxapi-0.1:clang-8: + extends: + - .variables:default + - .use-clang:base + stage: test + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + variables: + PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml + PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml + PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml + PY_MPI_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-mpi-JUnitTestResults.xml + EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" + script: + - source $INSTALL_DIR/bin/GMXRC + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash -x admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh + artifacts: + reports: + junit: + - $PY_UNIT_TEST_XML + - $PY_MPI_UNIT_TEST_XML + - $PY_ACCEPTANCE_TEST_XML + - $PY_MPI_ACCEPTANCE_TEST_XML + when: always + expire_in: 1 week + needs: + - job: gromacs:clang-8:build + artifacts: true + +gmxapi-0.1:clang-8:py-3.6.10: + extends: + - .gmxapi-0.1:clang-8 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.6" + PY_VER: "3.6.10" + +gmxapi-0.1:clang-8:py-3.7.7: + extends: + - .gmxapi-0.1:clang-8 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.7" + PY_VER: "3.7.7" + +gmxapi-0.1:clang-8:py-3.8.2: + extends: + - .gmxapi-0.1:clang-8 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.8" + PY_VER: "3.8.2" + +# Base job definition for gmxapi 0.2 tests against GROMACS 2021 in clang-8 environments. +.gmxapi-0.2:clang-8:gmx2021: + extends: + - .variables:default + - .use-clang:base + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + stage: test + variables: + PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml + PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml + PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml + PY_MPI_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-mpi-JUnitTestResults.xml + script: + - source $INSTALL_DIR/bin/GMXRC + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash -x admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh + artifacts: + reports: + junit: + - $PY_UNIT_TEST_XML + - $PY_MPI_UNIT_TEST_XML + - $PY_ACCEPTANCE_TEST_XML + - $PY_MPI_ACCEPTANCE_TEST_XML + when: always + expire_in: 1 week + needs: + - job: gromacs:clang-8:build + artifacts: true + +gmxapi-0.2:clang-8:gmx2021:py-3.6.10: + extends: + - .gmxapi-0.2:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.6" + PY_VER: "3.6.10" + +gmxapi-0.2:clang-8:gmx2021:py-3.7.7: + extends: + - .gmxapi-0.2:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.7" + PY_VER: "3.7.7" + +gmxapi-0.2:clang-8:gmx2021:py-3.8.2: + extends: + - .gmxapi-0.2:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.8" + PY_VER: "3.8.2" diff --git a/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml new file mode 100644 index 0000000000..2b6885eb4d --- /dev/null +++ b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml @@ -0,0 +1,53 @@ +# Test for API regression or unexpected breakage. + +# Base job definition for sample_restraint tests against GROMACS 2020. +.sample_restraint:clang-8:gmx2020: + extends: + - .variables:default + - .use-clang:base + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + stage: test + variables: + PLUGIN_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml + PLUGIN_MPI_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml + EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" + script: + - source $INSTALL_DIR/bin/GMXRC + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash -x admin/ci-scripts/build-and-test-sample_restraint-2020.sh + artifacts: + reports: + junit: + - $PLUGIN_TEST_XML + - $PLUGIN_MPI_TEST_XML + when: always + expire_in: 1 week + needs: + # Note: if we want to run regression tests using artifacts from the `release-2020` + # pipelines, we could specify *project* and *ref*, but initially we are only + # testing within a single repository. + - job: gromacs:clang-8:build + artifacts: true + +sample_restraint:clang-8:gmx2020:py-3.6.10: + extends: + - .sample_restraint:clang-8:gmx2020 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.6" + PY_VER: "3.6.10" + +sample_restraint:clang-8:gmx2020:py-3.7.7: + extends: + - .sample_restraint:clang-8:gmx2020 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.7" + PY_VER: "3.7.7" + +sample_restraint:clang-8:gmx2020:py-3.8.2: + extends: + - .sample_restraint:clang-8:gmx2020 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.8" + PY_VER: "3.8.2" diff --git a/admin/gitlab-ci/sample_restraint.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml new file mode 100644 index 0000000000..f4909bc61d --- /dev/null +++ b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml @@ -0,0 +1,48 @@ +# Base job definition for sample_restraint tests against GROMACS 2021. +.sample_restraint:clang-8:gmx2021: + extends: + - .variables:default + - .use-clang:base + image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 + stage: test + variables: + PLUGIN_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml + PLUGIN_MPI_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml + EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" + script: + - source $INSTALL_DIR/bin/GMXRC + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash -x admin/ci-scripts/build-and-test-sample_restraint-2020.sh + artifacts: + reports: + junit: + - $PLUGIN_TEST_XML + - $PLUGIN_MPI_TEST_XML + when: always + expire_in: 1 week + needs: + - job: gromacs:clang-8:build + artifacts: true + +sample_restraint:clang-8:gmx2021:py-3.6.10: + extends: + - .sample_restraint:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.6" + PY_VER: "3.6.10" + +sample_restraint:clang-8:gmx2021:py-3.7.7: + extends: + - .sample_restraint:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.7" + PY_VER: "3.7.7" + +sample_restraint:clang-8:gmx2021:py-3.8.2: + extends: + - .sample_restraint:clang-8:gmx2021 + - .rules:merge-requests + variables: + VENVPATH: "/root/venv/py3.8" + PY_VER: "3.8.2" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 427fce5f08..64506cbae1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ # # This file is part of the GROMACS molecular simulation package. # -# Copyright (c) 2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by +# Copyright (c) 2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. diff --git a/src/gromacs/CMakeLists.txt b/src/gromacs/CMakeLists.txt index c0adfafe2b..60a0fc186d 100644 --- a/src/gromacs/CMakeLists.txt +++ b/src/gromacs/CMakeLists.txt @@ -1,7 +1,8 @@ # # This file is part of the GROMACS molecular simulation package. # -# Copyright (c) 2010,2011,2012,2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by +# Copyright (c) 2010,2011,2012,2013,2014,2015, The GROMACS development team. +# Copyright (c) 2016,2017,2018,2019,2020, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. @@ -343,6 +344,13 @@ if (GMX_CLANG_TIDY) "${CLANG_TIDY_EXE};-warnings-as-errors=*") endif() +# clang-3.6 warns about a number of issues that are not reported by more modern compilers +# and we know they are not real issues. So we only check that it can compile without error +# but ignore all warnings. +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION MATCHES "^3\.6") + target_compile_options(libgromacs PRIVATE $<$:-w>) +endif() + # Only install the library in mdrun-only mode if it is actually necessary # for the binary if (NOT GMX_BUILD_MDRUN_ONLY OR BUILD_SHARED_LIBS) diff --git a/src/gromacs/fileio/matio.cpp b/src/gromacs/fileio/matio.cpp index dc223c6717..cbb9c96b79 100644 --- a/src/gromacs/fileio/matio.cpp +++ b/src/gromacs/fileio/matio.cpp @@ -432,6 +432,7 @@ static t_matrix read_xpm_entry(FILE* in) line = line_buf; } bSetLine = TRUE; + GMX_RELEASE_ASSERT(line, "Need to have valid line to parse"); if (strstr(line, "x-axis")) { line = std::strstr(line, "x-axis"); diff --git a/src/gromacs/selection/CMakeLists.txt b/src/gromacs/selection/CMakeLists.txt index c54967aad2..8adb34434a 100644 --- a/src/gromacs/selection/CMakeLists.txt +++ b/src/gromacs/selection/CMakeLists.txt @@ -1,7 +1,8 @@ # # This file is part of the GROMACS molecular simulation package. # -# Copyright (c) 2010,2012,2013,2014,2015,2018,2019, by the GROMACS development team, led by +# Copyright (c) 2010,2012,2013,2014,2015, The GROMACS development team. +# Copyright (c) 2018,2019,2020, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. @@ -53,6 +54,7 @@ else() gmx_target_warning_suppression(scanner -Wno-unused HAS_NO_UNUSED) gmx_target_warning_suppression(scanner -Wno-unused-parameter HAS_NO_UNUSED_PARAMETER) gmx_target_warning_suppression(scanner -Wno-missing-declarations HAS_NO_MISSING_DECLARATIONS) + gmx_target_warning_suppression(scanner -Wno-null-conversion HAS_NO_NULL_CONVERSIONS) gmx_target_warning_suppression(scanner -wd1419 HAS_DECL_IN_SOURCE) endif() list(APPEND libgromacs_object_library_dependencies scanner) diff --git a/src/testutils/TestMacros.cmake b/src/testutils/TestMacros.cmake index ca52ca973d..1c4e0d71af 100644 --- a/src/testutils/TestMacros.cmake +++ b/src/testutils/TestMacros.cmake @@ -1,7 +1,7 @@ # # This file is part of the GROMACS molecular simulation package. # -# Copyright (c) 2011,2012,2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by +# Copyright (c) 2011,2012,2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. @@ -41,6 +41,13 @@ function (gmx_add_unit_test_library NAME) target_compile_definitions(${NAME} PRIVATE HAVE_CONFIG_H) target_include_directories(${NAME} SYSTEM BEFORE PRIVATE ${PROJECT_SOURCE_DIR}/src/external/thread_mpi/include) target_link_libraries(${NAME} PRIVATE testutils gmock) + # clang-3.6 warns about a number of issues that are not reported by more modern compilers + # and we know they are not real issues. So we only check that it can compile without error + # but ignore all warnings. + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION MATCHES "^3\.6") + target_compile_options(${NAME} PRIVATE $<$:-w>) + endif() + endif() endfunction () @@ -94,6 +101,12 @@ function (gmx_add_gtest_executable EXENAME) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION MATCHES "^6\.0") target_compile_options(${EXENAME} PRIVATE $<$:-Weverything ${IGNORED_CLANG_ALL_WARNINGS} -Wno-gnu-zero-variadic-macro-arguments -Wno-zero-as-null-pointer-constant -Wno-missing-variable-declarations>) endif() + # clang-3.6 warns about a number of issues that are not reported by more modern compilers + # and we know they are not real issues. So we only check that it can compile without error + # but ignore all warnings. + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION MATCHES "^3\.6") + target_compile_options(${EXENAME} PRIVATE $<$:-w>) + endif() endif() endfunction() -- 2.22.0