From c6597e793525d27f00659d37d6186b3a964f7a36 Mon Sep 17 00:00:00 2001 From: Paul Bauer Date: Thu, 14 May 2020 09:53:03 +0000 Subject: [PATCH] Avoid CI errors from overloading resources CI got cause ICE due to resource limitations. Avoided by limiting parallelism for make. Change-Id: I6cc09f16625f615984e71c650eafa755abc1f121 --- admin/gitlab-ci/archive.gitlab-ci.yml | 10 +++++----- admin/gitlab-ci/documentation.gitlab-ci.yml | 6 +++--- admin/gitlab-ci/global.gitlab-ci.yml | 2 +- admin/gitlab-ci/gromacs.gitlab-ci.yml | 17 ++++++++++------- admin/gitlab-ci/lint.gitlab-ci.yml | 10 +++++----- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index cac89d8b98..6c71b7a6d7 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -12,7 +12,7 @@ prepare-release-version: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi script: - cmake -P cmake/gmxVersionInfo.cmake &> version.json artifacts: @@ -32,7 +32,7 @@ regressiontests:prepare: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi # Always clone the default version for this branch, master in this case script: - export REGTESTBRANCH=master @@ -67,7 +67,7 @@ regressiontests:package: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi needs: - job: prepare-release-version script: @@ -140,7 +140,7 @@ archive:configure:release: variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 - KUBERNETES_MEMORY_LIMIT: 4Gi + KUBERNETES_MEMORY_REQUEST: 4Gi script: - rm -rf gromacs*tar.gz - cd $BUILD_DIR @@ -209,7 +209,7 @@ release-verify: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi script: - VERSION=`cat version.json | python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"` diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 35a8235b56..25a4c68133 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -12,7 +12,7 @@ variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7" CMAKE_EXTRA_OPTIONS: "" CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" @@ -79,7 +79,7 @@ variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7" CMAKE_EXTRA_OPTIONS: "" CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" @@ -229,7 +229,7 @@ docs:build: variables: KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 - KUBERNETES_MEMORY_LIMIT: 4Gi + KUBERNETES_MEMORY_REQUEST: 4Gi script: - cd $BUILD_DIR - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml index ef488a3c6a..2f56fc3045 100644 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: KUBERNETES_CPU_LIMIT: 8 KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_LIMIT: 8Gi + KUBERNETES_MEMORY_REQUEST: 8Gi KUBERNETES_EXTENDED_RESOURCE_NAME: "" KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index baf9fc3dc9..6780f6d5da 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -12,7 +12,7 @@ simple-build: variables: KUBERNETES_CPU_LIMIT: 8 KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_LIMIT: 8Gi + KUBERNETES_MEMORY_REQUEST: 8Gi CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None" CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON" CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF" @@ -52,8 +52,8 @@ simple-build: 2>&1 | tee cmakeLog.log - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log - if [ -s cmakeErrors.log ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi - - cmake --build . -- -j8 2>&1 | tee buildLogFile.log - - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log + - cmake --build . -- -j$KUBERNETES_CPU_LIMIT 2>&1 | tee buildLogFile.log + - cmake --build . --target tests -- -j$KUBERNETES_CPU_LIMIT 2>&1 | tee testBuildLogFile.log - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true @@ -86,7 +86,7 @@ simple-build: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi CMAKE_COMPILER_SCRIPT: "" CMAKE_EXTRA_OPTIONS: "" CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" @@ -138,7 +138,7 @@ simple-build: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi CMAKE_COMPILER_SCRIPT: "" CMAKE_EXTRA_OPTIONS: "" CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AUTO" @@ -359,8 +359,8 @@ gromacs:clang-8-cuda-10.1:release:configure: stage: build script: - cd $BUILD_DIR - - cmake --build . -- -j8 2>&1 | tee buildLogFile.log - - cmake --build . --target tests -- -j8 2>&1 | tee testBuildLogFile.log + - cmake --build . -- -j$KUBERNETES_CPU_LIMIT 2>&1 | tee buildLogFile.log + - cmake --build . --target tests -- -j$KUBERNETES_CPU_LIMIT 2>&1 | tee testBuildLogFile.log - awk '/warning/,/warning.*generated|^$/' buildLogFile.log testBuildLogFile.log | grep -v "CMake" | tee buildErrors.log || true - grep "cannot be built" buildLogFile.log testBuildLogFile.log | tee -a buildErrors.log || true @@ -564,6 +564,9 @@ gromacs:clang-8-cuda-10.1:release:build: extends: - .variables:default variables: + KUBERNETES_CPU_LIMIT: 4 + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_REQUEST: 4Gi REGRESSIONTEST_TOTAL_RANK_NUMBER: 4 REGRESSIONTEST_OMP_RANK_NUMBER: 2 REGRESSIONTEST_PME_RANK_NUMBER: 1 diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index d6a24c0f91..a283c5f5d6 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -50,7 +50,7 @@ clang-tidy:test: EXTRA_INSTALLS: clang-tidy-$COMPILER_MAJOR_VERSION KUBERNETES_CPU_LIMIT: 4 KUBERNETES_CPU_REQUEST: 2 - KUBERNETES_MEMORY_LIMIT: 4Gi + KUBERNETES_MEMORY_REQUEST: 4Gi script: # TODO (issue #3272) `master` is not appropriate for use on release-xxxx branches, how should we handle that? - REV=$(git fetch -q https://gitlab.com/gromacs/gromacs.git master && git show -s --pretty=format:"%h" `git merge-base FETCH_HEAD HEAD`) @@ -80,7 +80,7 @@ clang-format: COMPILER_MAJOR_VERSION: 7 KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi EXTRA_INSTALLS: clang-format-$COMPILER_MAJOR_VERSION script: # TODO (issue #3272) `master` is not appropriate for use on release-xxxx branches, how should we handle that? @@ -110,7 +110,7 @@ copyright-check: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi script: # TODO (issue #3272) `master` is not appropriate for use on release-xxxx branches, how should we handle that? - REV=$(git fetch -q https://gitlab.com/gromacs/gromacs.git master && git show -s --pretty=format:"%h" `git merge-base FETCH_HEAD HEAD`) @@ -148,7 +148,7 @@ check-source: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi BUILD_DIR: build-docs script: - cd $BUILD_DIR @@ -177,7 +177,7 @@ linkchecker: variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi BUILD_DIR: build-docs script: - cd $BUILD_DIR -- 2.22.0