From ba3bd95d346fe5a1dba3b04b9f36c11302b0036c Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Thu, 19 Mar 2020 13:15:46 +0300 Subject: [PATCH] Consolidate global mix-ins. Add a generic documentation section for CI job parameters and add global.gitlab-ci.yml with mix-in job definitions. Change-Id: I4e34845deed69e949e849d6a4a34462965f1646f --- .gitlab-ci.yml | 17 +++++-- admin/ci-templates/.build-cache-template.yml | 5 -- admin/ci-templates/.pull-cache-template.yml | 8 --- admin/ci-templates/.variables-template.yml | 11 ---- admin/gitlab-ci/documentation.gitlab-ci.yml | 2 +- .../global.gitlab-ci.yml} | 28 ++++++++++ admin/gitlab-ci/lint.gitlab-ci.yml | 6 +-- admin/gitlab-ci/testing-matrix.gitlab-ci.yml | 12 ++--- docs/dev-manual/gitlab.rst | 51 +++++++++++++++++-- 9 files changed, 97 insertions(+), 43 deletions(-) delete mode 100644 admin/ci-templates/.build-cache-template.yml delete mode 100644 admin/ci-templates/.pull-cache-template.yml delete mode 100644 admin/ci-templates/.variables-template.yml rename admin/{ci-templates/.image-templates.yml => gitlab-ci/global.gitlab-ci.yml} (60%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e0c234658..e64b7f1c65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,11 +40,20 @@ default: tags: - k8s-scilifelab +variables: + KUBERNETES_CPU_LIMIT: 8 + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_LIMIT: 8Gi + KUBERNETES_EXTENDED_RESOURCE_NAME: "" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 + CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" + BUILD_DIR: build + INSTALL_DIR: install + include: - - local: '/admin/ci-templates/.image-templates.yml' - - local: '/admin/ci-templates/.build-cache-template.yml' - - local: '/admin/ci-templates/.pull-cache-template.yml' - - local: '/admin/ci-templates/.variables-template.yml' + # Mix-in helper job definitions of general utility. + - local: '/admin/gitlab-ci/global.gitlab-ci.yml' + # Various composable job definitions. - local: '/admin/ci-templates/.build-template.yml' - local: '/admin/ci-templates/.test-runner-template.yml' - local: '/admin/ci-templates/.gcc-before-script.yml' diff --git a/admin/ci-templates/.build-cache-template.yml b/admin/ci-templates/.build-cache-template.yml deleted file mode 100644 index c65c8333e3..0000000000 --- a/admin/ci-templates/.build-cache-template.yml +++ /dev/null @@ -1,5 +0,0 @@ -.build-cache-template: - cache: - key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - ccache/ diff --git a/admin/ci-templates/.pull-cache-template.yml b/admin/ci-templates/.pull-cache-template.yml deleted file mode 100644 index b35be1a138..0000000000 --- a/admin/ci-templates/.pull-cache-template.yml +++ /dev/null @@ -1,8 +0,0 @@ -.pull-cache-template: - cache: - key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - # Read but don't update the cache - policy: pull - -.no-cache-template: - cache: {} diff --git a/admin/ci-templates/.variables-template.yml b/admin/ci-templates/.variables-template.yml deleted file mode 100644 index b3d919a423..0000000000 --- a/admin/ci-templates/.variables-template.yml +++ /dev/null @@ -1,11 +0,0 @@ -.variables-template: - variables: - KUBERNETES_CPU_LIMIT: 8 - KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_LIMIT: 8Gi - KUBERNETES_EXTENDED_RESOURCE_NAME: "" - KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 - CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" - BUILD_DIR: build - INSTALL_DIR: install - diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 2e8b52168c..de262186dd 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -22,7 +22,7 @@ .build-docs-webpage: extends: - .binary-build-template - - .variables-template + - .variables:default - .no-cache-template - .build-docs-webpage-template - .documentation-before-script-template diff --git a/admin/ci-templates/.image-templates.yml b/admin/gitlab-ci/global.gitlab-ci.yml similarity index 60% rename from admin/ci-templates/.image-templates.yml rename to admin/gitlab-ci/global.gitlab-ci.yml index 084c703c25..127d9c2f5a 100644 --- a/admin/ci-templates/.image-templates.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -1,3 +1,31 @@ +.variables:default: + variables: + KUBERNETES_CPU_LIMIT: 8 + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_LIMIT: 8Gi + KUBERNETES_EXTENDED_RESOURCE_NAME: "" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 + CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" + BUILD_DIR: build + INSTALL_DIR: install + +.build-cache-template: + cache: + key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - ccache/ + +.pull-cache-template: + cache: + key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + # Read but don't update the cache + policy: pull + +.no-cache-template: + cache: {} + + +# Tool chains .gcc5-template: image: gromacs/ci-gcc-5:master diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index 1b12c5d628..b61293751b 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -22,7 +22,7 @@ run-clang-format: extends: - .binary-build-template - .no-cache-template - - .variables-template + - .variables:default - .clang-before-script-template - .build-clang-template - .clang7-template @@ -55,7 +55,7 @@ run-copyright-check: extends: - .binary-build-template - .no-cache-template - - .variables-template + - .variables:default - .clang-before-script-template - .build-clang-template - .clang7-template @@ -89,7 +89,7 @@ run-copyright-check: run-check-source: extends: - .no-cache-template - - .variables-template + - .variables:default - .build-docs-binary-template - .documentation-before-script-template stage: source-check diff --git a/admin/gitlab-ci/testing-matrix.gitlab-ci.yml b/admin/gitlab-ci/testing-matrix.gitlab-ci.yml index 694bed2559..4ce279a033 100644 --- a/admin/gitlab-ci/testing-matrix.gitlab-ci.yml +++ b/admin/gitlab-ci/testing-matrix.gitlab-ci.yml @@ -3,31 +3,31 @@ extends: - .configure-build-template - .no-cache-template - - .variables-template + - .variables:default .configure-release-extends-template: extends: - .configure-build-release-template - .no-cache-template - - .variables-template + - .variables:default .build-extends-template: extends: - .binary-build-template - .build-cache-template - - .variables-template + - .variables:default .test-extends-template: extends: - .test-base-template - .pull-cache-template - .test-script-template - - .variables-template + - .variables:default .regressiontests-extends-template: extends: - .no-cache-template - - .variables-template + - .variables:default # Templates for configuration stage @@ -120,7 +120,7 @@ simple-build: extends: - .simple-build-template - .build-cache-template - - .variables-template + - .variables:default - .gcc-before-script-template - .build-gcc-template - .gcc7-template diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst index ad47179f83..3643ad0597 100644 --- a/docs/dev-manual/gitlab.rst +++ b/docs/dev-manual/gitlab.rst @@ -28,16 +28,57 @@ Pipeline execution .. todo:: Comment on the number of pipelines that can be or which are likely to be running at the same time. +Configuration files +~~~~~~~~~~~~~~~~~~~ + +At the root of the repository, :file:`.gitlab-ci.yml` defines the stages and +some default parameters, then includes files from :file:`admin/gitlab-ci/` to +define jobs to be executed in the pipelines. + +Note that job names beginning with a period (``.``) are +`"hidden" `_. +Such jobs are not directly eligible to run, but may be used as templates +via the `*extends* job property `_. + +Job parameters +~~~~~~~~~~~~~~ + +Refer to https://docs.gitlab.com/ee/ci/yaml for complete documentation on +GitLab CI job parameters, but note the following GROMACS-specific conventions. + +.. glossary:: + + before_script + Used by several of our templates to prepend shell commands to + a job *script* parameter. + Avoid using *before-script* directly, and be cautious + about nested *extends* overriding multiple *before_script* definitions. + + image + Part of the tool chain configuration. Instead of setting *image* + directly, *extend* a *.use_* template from + :file:`admin/gitlab-ci/global.gitlab-ci.yml` + + variables + Many job definitions will add or override keys in *variables*. + Refer to `GitLab `__ + for details of the merging behavior. Refer to :ref:`variables` for local usage. + +In addition to the templates in the main job definition files, +common "mix-in" functionality and behavioral templates are defined in +:file:`admin/gitlab-ci/global.gitlab-ci.yml`. + +.. _variables: + Variables ~~~~~~~~~ The GitLab CI framework, GitLab Runner, plugins, and our own scripts set and -use several variables (usually as a key under a *variables* parameter in -the YAML configuration). +use several `variables `__. -Some default values are specified for all jobs in :file:`.gitlab-ci.yml`. -Many of the mix-in / template jobs in :file:`admin/gitlab-ci/global.gitlab-ci.yml` -provide additional or overriding definitions. +Default values are available from the ``.variables:default`` definition in +:file:`admin/gitlab-ci/global.gitlab-ci.yml`. +Many of the mix-in / template jobs provide additional or overriding definitions. Other variables may be set when making final job definitions. Variables may control the behvior of GitLab-CI (those beginning with ``CI_``), -- 2.22.0