Consolidate global mix-ins.

Add a generic documentation section for CI job parameters
and add global.gitlab-ci.yml with mix-in job definitions.

Change-Id: I4e34845deed69e949e849d6a4a34462965f1646f

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7e0c2346582f9a176fa6c670ab1a0049719349a2..e64b7f1c654b2a0984b6b784bf60a1b2d95893ec 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,11 +40,20 @@ default:
   tags:
     - k8s-scilifelab
 
+variables:
+  KUBERNETES_CPU_LIMIT: 8
+  KUBERNETES_CPU_REQUEST: 4
+  KUBERNETES_MEMORY_LIMIT: 8Gi
+  KUBERNETES_EXTENDED_RESOURCE_NAME: ""
+  KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0
+  CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master"
+  BUILD_DIR: build
+  INSTALL_DIR: install
+
 include:
-  - local: '/admin/ci-templates/.image-templates.yml'
-  - local: '/admin/ci-templates/.build-cache-template.yml'
-  - local: '/admin/ci-templates/.pull-cache-template.yml'
-  - local: '/admin/ci-templates/.variables-template.yml'
+  # Mix-in helper job definitions of general utility.
+  - local: '/admin/gitlab-ci/global.gitlab-ci.yml'
+  # Various composable job definitions.
   - local: '/admin/ci-templates/.build-template.yml'
   - local: '/admin/ci-templates/.test-runner-template.yml'
   - local: '/admin/ci-templates/.gcc-before-script.yml'

diff --git a/admin/ci-templates/.build-cache-template.yml b/admin/ci-templates/.build-cache-template.yml
deleted file mode 100644 (file)
index c65c833..0000000
--- a/admin/ci-templates/.build-cache-template.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-.build-cache-template:
-    cache:
-      key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
-      paths:
-        - ccache/

diff --git a/admin/ci-templates/.pull-cache-template.yml b/admin/ci-templates/.pull-cache-template.yml
deleted file mode 100644 (file)
index b35be1a..0000000
--- a/admin/ci-templates/.pull-cache-template.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-.pull-cache-template:
-    cache:
-      key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
-      # Read but don't update the cache
-      policy: pull
-
-.no-cache-template:
-    cache: {}

diff --git a/admin/ci-templates/.variables-template.yml b/admin/ci-templates/.variables-template.yml
deleted file mode 100644 (file)
index b3d919a..0000000
--- a/admin/ci-templates/.variables-template.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-.variables-template:
-    variables:
-     KUBERNETES_CPU_LIMIT: 8
-     KUBERNETES_CPU_REQUEST: 4
-     KUBERNETES_MEMORY_LIMIT: 8Gi
-     KUBERNETES_EXTENDED_RESOURCE_NAME: ""
-     KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0
-     CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master"
-     BUILD_DIR: build
-     INSTALL_DIR: install
-

diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml
index 2e8b52168cee10416da3f57fb6e81151475c3668..de262186ddf401c633aa2098e6bdf726a60bd3a9 100644 (file)
--- a/admin/gitlab-ci/documentation.gitlab-ci.yml
+++ b/admin/gitlab-ci/documentation.gitlab-ci.yml
@@ -22,7 +22,7 @@
 .build-docs-webpage:
   extends:
     - .binary-build-template
-    - .variables-template
+    - .variables:default
     - .no-cache-template
     - .build-docs-webpage-template
     - .documentation-before-script-template

diff --git a/admin/ci-templates/.image-templates.yml b/admin/gitlab-ci/global.gitlab-ci.yml
similarity index 60%
rename from admin/ci-templates/.image-templates.yml
rename to admin/gitlab-ci/global.gitlab-ci.yml
index 084c703c255dd6d7a5e444900afc1da91817bf86..127d9c2f5a43898d08aa590a12cc47bfe19f8c10 100644 (file)
--- a/admin/ci-templates/.image-templates.yml
+++ b/admin/gitlab-ci/global.gitlab-ci.yml
@@ -1,3 +1,31 @@
+.variables:default:
+  variables:
+    KUBERNETES_CPU_LIMIT: 8
+    KUBERNETES_CPU_REQUEST: 4
+    KUBERNETES_MEMORY_LIMIT: 8Gi
+    KUBERNETES_EXTENDED_RESOURCE_NAME: ""
+    KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0
+    CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master"
+    BUILD_DIR: build
+    INSTALL_DIR: install
+
+.build-cache-template:
+  cache:
+    key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
+    paths:
+      - ccache/
+
+.pull-cache-template:
+  cache:
+    key: "$CI_JOB_NAME-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
+    # Read but don't update the cache
+    policy: pull
+
+.no-cache-template:
+  cache: {}
+
+
+# Tool chains
 .gcc5-template:
   image: gromacs/ci-gcc-5:master
 

diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml
index 1b12c5d628df2f6d1ea7e8e20de101f16fde4a9a..b61293751b3d9f0f0ec9f0cff8b3ea5d57d8a4bf 100644 (file)
--- a/admin/gitlab-ci/lint.gitlab-ci.yml
+++ b/admin/gitlab-ci/lint.gitlab-ci.yml
@@ -22,7 +22,7 @@ run-clang-format:
   extends:
     - .binary-build-template
     - .no-cache-template
-    - .variables-template
+    - .variables:default
     - .clang-before-script-template
     - .build-clang-template
     - .clang7-template
@@ -55,7 +55,7 @@ run-copyright-check:
   extends:
     - .binary-build-template
     - .no-cache-template
-    - .variables-template
+    - .variables:default
     - .clang-before-script-template
     - .build-clang-template
     - .clang7-template
@@ -89,7 +89,7 @@ run-copyright-check:
 run-check-source:
   extends:
     - .no-cache-template
-    - .variables-template
+    - .variables:default
     - .build-docs-binary-template
     - .documentation-before-script-template
   stage: source-check

diff --git a/admin/gitlab-ci/testing-matrix.gitlab-ci.yml b/admin/gitlab-ci/testing-matrix.gitlab-ci.yml
index 694bed25592f40d26702a430d4785b1ec137baf1..4ce279a033ceb36b47982448f0bbbfc54b6c5fa6 100644 (file)
--- a/admin/gitlab-ci/testing-matrix.gitlab-ci.yml
+++ b/admin/gitlab-ci/testing-matrix.gitlab-ci.yml
@@ -3,31 +3,31 @@
   extends:
     - .configure-build-template
     - .no-cache-template
-    - .variables-template
+    - .variables:default
 
 .configure-release-extends-template:
   extends:
     - .configure-build-release-template
     - .no-cache-template
-    - .variables-template
+    - .variables:default
 
 .build-extends-template:
   extends:
     - .binary-build-template
     - .build-cache-template
-    - .variables-template
+    - .variables:default
 
 .test-extends-template:
   extends:
     - .test-base-template
     - .pull-cache-template
     - .test-script-template
-    - .variables-template
+    - .variables:default
 
 .regressiontests-extends-template:
   extends:
     - .no-cache-template
-    - .variables-template
+    - .variables:default
 
 
 # Templates for configuration stage
@@ -120,7 +120,7 @@ simple-build:
   extends:
     - .simple-build-template
     - .build-cache-template
-    - .variables-template
+    - .variables:default
     - .gcc-before-script-template
     - .build-gcc-template
     - .gcc7-template

diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst
index ad47179f83d47e4e4d7480e23f0b6711937cd11c..3643ad059713cda1594a67e8f4de74876da99a43 100644 (file)
--- a/docs/dev-manual/gitlab.rst
+++ b/docs/dev-manual/gitlab.rst
@@ -28,16 +28,57 @@ Pipeline execution
 
 .. todo:: Comment on the number of pipelines that can be or which are likely to be running at the same time.
 
+Configuration files
+~~~~~~~~~~~~~~~~~~~
+
+At the root of the repository, :file:`.gitlab-ci.yml` defines the stages and
+some default parameters, then includes files from :file:`admin/gitlab-ci/` to
+define jobs to be executed in the pipelines.
+
+Note that job names beginning with a period (``.``) are
+`"hidden" <https://docs.gitlab.com/ee/ci/yaml/#hidden-keys-jobs>`_.
+Such jobs are not directly eligible to run, but may be used as templates
+via the `*extends* job property <https://docs.gitlab.com/ee/ci/yaml/#extends>`_.
+
+Job parameters
+~~~~~~~~~~~~~~
+
+Refer to https://docs.gitlab.com/ee/ci/yaml for complete documentation on
+GitLab CI job parameters, but note the following GROMACS-specific conventions.
+
+.. glossary::
+
+    before_script
+        Used by several of our templates to prepend shell commands to
+        a job *script* parameter.
+        Avoid using *before-script* directly, and be cautious
+        about nested *extends* overriding multiple *before_script* definitions.
+
+    image
+        Part of the tool chain configuration. Instead of setting *image*
+        directly, *extend* a *.use_<toolchain>* template from
+        :file:`admin/gitlab-ci/global.gitlab-ci.yml`
+
+    variables
+        Many job definitions will add or override keys in *variables*.
+        Refer to `GitLab <https://docs.gitlab.com/ee/ci/yaml/#variables>`__
+        for details of the merging behavior. Refer to :ref:`variables` for local usage.
+
+In addition to the templates in the main job definition files,
+common "mix-in" functionality and behavioral templates are defined in
+:file:`admin/gitlab-ci/global.gitlab-ci.yml`.
+
+.. _variables:
+
 Variables
 ~~~~~~~~~
 
 The GitLab CI framework, GitLab Runner, plugins, and our own scripts set and
-use several variables (usually as a key under a *variables* parameter in
-the YAML configuration).
+use several `variables <https://docs.gitlab.com/ee/ci/variables/README.html>`__.
 
-Some default values are specified for all jobs in :file:`.gitlab-ci.yml`.
-Many of the mix-in / template jobs in :file:`admin/gitlab-ci/global.gitlab-ci.yml`
-provide additional or overriding definitions.
+Default values are available from the ``.variables:default`` definition in
+:file:`admin/gitlab-ci/global.gitlab-ci.yml`.
+Many of the mix-in / template jobs provide additional or overriding definitions.
 Other variables may be set when making final job definitions.
 
 Variables may control the behvior of GitLab-CI (those beginning with ``CI_``),