From 4d175f0baf4cd35028443854fb4ca60ab7e97527 Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Fri, 23 Oct 2020 14:09:50 +0000 Subject: [PATCH] Update Docker image build scripts. Adopt new naming scheme and conventions for Docker image contents. Jobs will be updated to use the new images in a follow-up change. Refs #3621 --- admin/containers/buildall.sh | 63 +++++-------- .../containers/scripted_gmx_docker_builds.py | 88 ++++++++++-------- admin/containers/utility.py | 93 +++++++++++++++++-- docs/dev-manual/containers.rst | 17 ++-- docs/dev-manual/gitlab.rst | 41 ++++---- 5 files changed, 187 insertions(+), 115 deletions(-) diff --git a/admin/containers/buildall.sh b/admin/containers/buildall.sh index 1d2daad833..35d37d8a69 100644 --- a/admin/containers/buildall.sh +++ b/admin/containers/buildall.sh @@ -9,46 +9,29 @@ SCRIPT=$PWD/scripted_gmx_docker_builds.py # images needed, because the same one can test library, # thread and no MPI configurations. -tag="gromacs/cmake-3.15.7-gcc-8-cuda-11.0-nvidiaopencl-clfft-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.15.7 --gcc 8 --cuda 11.0 --opencl --clfft --mpi openmpi \ -| docker build -t $tag - - -tag="gromacs/cmake-3.13.0-gcc-7-amdopencl-clfft-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.13.0 --gcc 7 --opencl amd --clfft --mpi openmpi --ubuntu 18.04 | docker build -t $tag - - -tag="gromacs/cmake-3.13.0-llvm-8-tsan-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.13.0 --llvm 8 --tsan | docker build -t $tag - - -tag="gromacs/cmake-3.15.7-llvm-8-cuda-10.0-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.15.7 --llvm 8 --cuda 10.0 --mpi openmpi | docker build -t $tag - - -tag="gromacs/cmake-3.15.7-llvm-8-cuda-11.0-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.15.7 --llvm 8 --cuda 11.0 --mpi openmpi | docker build -t $tag - - -tag="gromacs/cmake-3.15.7-llvm-9-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.15.7 --llvm 9 --mpi openmpi | docker build -t $tag - - -tag="gromacs/cmake-3.13.0-llvm-9-intelopencl-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.13.0 --llvm 9 --opencl intel --mpi openmpi | docker build -t $tag - - -tag="gromacs/cmake-3.13.0-llvm-9-amdopencl-openmpi-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.13.0 --llvm 9 --opencl amd --mpi openmpi --ubuntu 18.04 | docker build -t $tag - - -tag="gromacs/cmake-3.17.2-oneapi-2021.1-beta09-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.17.2 --oneapi 2021.1-beta09 | docker build -t $tag - - -tag="gromacs/ci-docs-llvm-master" -tags[${#tags[@]}]=$tag -python3 $SCRIPT --cmake 3.17.2 --llvm --doxygen | docker build -t $tag - +args[${#args[@]}]="--gcc 8 --cuda 11.0 --clfft --mpi openmpi" +args[${#args[@]}]="--gcc 7 --clfft --mpi openmpi --ubuntu 18.04" +args[${#args[@]}]="--llvm 8 --tsan" +args[${#args[@]}]="--llvm 8 --cuda 10.0 --clfft --mpi openmpi" +args[${#args[@]}]="--llvm 8 --cuda 11.0 --clfft --mpi openmpi" +args[${#args[@]}]="--llvm 9 --clfft --mpi openmpi --ubuntu 18.04" +args[${#args[@]}]="--oneapi 2021.1-beta09" +args[${#args[@]}]="--llvm --doxygen" + +echo "Building the following images." +for arg_string in "${args[@]}"; do + # shellcheck disable=SC2086 + python3 -m utility $arg_string +done +echo + +for arg_string in "${args[@]}"; do + # shellcheck disable=SC2086 + tag=$(python3 -m utility $arg_string) + tags[${#tags[@]}]=$tag + # shellcheck disable=SC2086 + python3 $SCRIPT $arg_string | docker build -t $tag - +done echo "Run the following to upload the updated images." echo "docker login" diff --git a/admin/containers/scripted_gmx_docker_builds.py b/admin/containers/scripted_gmx_docker_builds.py index a7b155ac01..5464a7f17c 100755 --- a/admin/containers/scripted_gmx_docker_builds.py +++ b/admin/containers/scripted_gmx_docker_builds.py @@ -33,7 +33,8 @@ # To help us fund GROMACS development, we humbly ask that you cite # the research papers on the package. Check out http://www.gromacs.org. -""" +"""Building block based Dockerfile generation for CI testing images. + Generates a set of docker images used for running GROMACS CI on Gitlab. The images are prepared according to a selection of build configuration targets that hope to cover a broad enough scope of different possible systems, @@ -44,6 +45,9 @@ analysing the logic and adding build stages as needed. Based on the example script provided by the NVidia HPCCM repository. +Reference: + `NVidia HPC Container Maker `__ + Authors: * Paul Bauer * Eric Irrgang @@ -56,6 +60,9 @@ Usage:: $ python3 scripted_gmx_docker_builds.py --format docker > Dockerfile && docker build . $ python3 scripted_gmx_docker_builds.py | docker build - +See Also: + :file:`buildall.sh` + """ import argparse @@ -80,6 +87,7 @@ _common_packages = ['build-essential', 'ccache', 'git', 'gnupg', + 'gpg-agent', 'libfftw3-dev', 'libhwloc-dev', 'liblapack-dev', @@ -92,6 +100,22 @@ _common_packages = ['build-essential', 'wget', 'xsltproc'] +_opencl_extra_packages = [ + 'nvidia-opencl-dev', + # The following require apt_ppas=['ppa:intel-opencl/intel-opencl'] + 'intel-opencl-icd', + 'ocl-icd-libopencl1', + 'ocl-icd-opencl-dev', + 'opencl-headers', + # The following require + # apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'], + # apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main'] + 'libelf1', + 'rocm-opencl', + 'rocm-dev', + 'clinfo' +] + # Extra packages needed to build Python installations from source. _python_extra_packages = ['build-essential', 'ca-certificates', @@ -134,18 +158,13 @@ _docs_extra_packages = ['autoconf', 'texlive-fonts-recommended', 'texlive-fonts-extra'] -# Supported Python versions for maintained branches. -_python_versions = ['3.6.10', '3.7.7', '3.8.2'] - # Parse command line arguments -parser = argparse.ArgumentParser(description='GROMACS CI image creation script', parents=[utility.parser]) +parser = argparse.ArgumentParser(description='GROMACS CI image creation script', + parents=[utility.parser]) parser.add_argument('--format', type=str, default='docker', choices=['docker', 'singularity'], help='Container specification format (default: docker)') -parser.add_argument('--venvs', nargs='*', type=str, default=_python_versions, - help='List of Python versions ("major.minor.patch") for which to install venvs. ' - 'Default: {}'.format(' '.join(_python_versions))) def base_image_tag(args) -> str: @@ -181,6 +200,13 @@ def get_llvm_packages(args) -> typing.Iterable[str]: return [] +def get_opencl_packages(args) -> typing.Iterable[str]: + if (args.doxygen is None) and (args.oneapi is None): + return _opencl_extra_packages + else: + return [] + + def get_compiler(args, compiler_build_stage: hpccm.Stage = None) -> bb_base: # Compiler if args.icc is not None: @@ -247,34 +273,6 @@ def get_mpi(args, compiler): return None -def get_opencl(args): - # Add OpenCL environment if needed - if (args.opencl is not None): - if args.opencl == 'nvidia': - if (args.cuda is None): - raise RuntimeError('Need Nvidia environment for Nvidia OpenCL image') - - return hpccm.building_blocks.packages(ospackages=['nvidia-opencl-dev']) - - elif args.opencl == 'intel': - # Note, when using oneapi, there is bundled OpenCL support, so this - # installation is not needed. - return hpccm.building_blocks.packages( - apt_ppas=['ppa:intel-opencl/intel-opencl'], - ospackages=['opencl-headers', 'ocl-icd-libopencl1', - 'ocl-icd-opencl-dev', 'intel-opencl-icd']) - - elif args.opencl == 'amd': - # libelf1 is a necessary dependency for something in the ROCm stack, - # which they should set up, but seem to have omitted. - return hpccm.building_blocks.packages( - apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'], - apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main'], - ospackages=['ocl-icd-libopencl1', 'ocl-icd-opencl-dev', 'opencl-headers', 'libelf1', 'rocm-opencl', 'rocm-dev', 'clinfo']) - else: - return None - - def get_clfft(args): if (args.clfft is not None): return hpccm.building_blocks.generic_cmake( @@ -538,21 +536,31 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]: # Building blocks are chunks of container-builder instructions that can be # copied to any build stage with the addition operator. building_blocks = collections.OrderedDict() + building_blocks['base_packages'] = hpccm.building_blocks.packages( + ospackages=_common_packages) # These are the most expensive and most reusable layers, so we put them first. building_blocks['compiler'] = get_compiler(args, compiler_build_stage=stages.get('compiler_build')) building_blocks['mpi'] = get_mpi(args, building_blocks['compiler']) + for i, cmake in enumerate(args.cmake): + building_blocks['cmake' + str(i)] = hpccm.building_blocks.cmake( + eula=True, + prefix='/usr/local/cmake-{}'.format(cmake), + version=cmake) # Install additional packages early in the build to optimize Docker build layer cache. - os_packages = _common_packages + get_llvm_packages(args) + os_packages = list(get_llvm_packages(args)) + get_opencl_packages(args) if args.doxygen is not None: os_packages += _docs_extra_packages if args.oneapi is not None: os_packages += ['lsb-release'] - building_blocks['ospackages'] = hpccm.building_blocks.packages(ospackages=os_packages) + building_blocks['extra_packages'] = hpccm.building_blocks.packages( + ospackages=os_packages, + apt_ppas=['ppa:intel-opencl/intel-opencl'], + apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'], + apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main'] + ) - building_blocks['cmake'] = hpccm.building_blocks.cmake(eula=True, version=args.cmake) - building_blocks['opencl'] = get_opencl(args) building_blocks['clfft'] = get_clfft(args) # Add Python environments to MPI images, only, so we don't have to worry diff --git a/admin/containers/utility.py b/admin/containers/utility.py index 5223e886f0..3df5fdc8be 100644 --- a/admin/containers/utility.py +++ b/admin/containers/utility.py @@ -32,9 +32,37 @@ # To help us fund GROMACS development, we humbly ask that you cite # the research papers on the package. Check out http://www.gromacs.org. -"""A `utility` module helps manage the matrix of configurations for CI testing and build containers. +"""A utility module to help manage the matrix of configurations for CI testing and build containers. -Provides importable argument parser. +When called as a stand alone script, prints a Docker image name based on the +command line arguments. The Docker image name is of the form used in the GROMACS +CI pipeline jobs. + +Example:: + + $ python3 -m utility --llvm --doxygen + gromacs/ci-ubuntu-18.04-llvm-7-docs + +See Also: + :file:`buildall.sh` + +As a module, provides importable argument parser and docker image name generator. + +Note that the parser is created with ``add_help=False`` to make it friendly as a +parent parser, but this means that you must derive a new parser from it if you +want to see the full generated command line help. + +Example:: + + import utility.parser + # utility.parser does not support `-h` or `--help` + parser = argparse.ArgumentParser( + description='GROMACS CI image creation script', + parents=[utility.parser]) + # ArgumentParser(add_help=True) is default, so parser supports `-h` and `--help` + +See Also: + :file:`scripted_gmx_docker_builds.py` Authors: * Paul Bauer @@ -58,8 +86,9 @@ parsers for tools. Instead, inherit from it with the *parents* argument to :py:class:`argparse.ArgumentParser` """ -parser.add_argument('--cmake', type=str, default='3.13.0', +parser.add_argument('--cmake', nargs='*', type=str, default=['3.13.0', '3.15.7', '3.17.2'], help='Selection of CMake version to provide to base image') + compiler_group = parser.add_mutually_exclusive_group() compiler_group.add_argument('--gcc', type=int, nargs='?', const=7, default=7, help='Select GNU compiler tool chain. (Default) ' @@ -94,11 +123,63 @@ parser.add_argument('--mpi', type=str, nargs='?', const='openmpi', default=None, parser.add_argument('--tsan', type=str, nargs='?', const='llvm', default=None, help='Build special compiler versions with TSAN OpenMP support') -parser.add_argument('--opencl', type=str, nargs='?', const='nvidia', default=None, - help='Provide environment for OpenCL builds') - parser.add_argument('--clfft', type=str, nargs='?', const='master', default=None, help='Add external clFFT libraries to the build image') parser.add_argument('--doxygen', type=str, nargs='?', const='1.8.5', default=None, help='Add doxygen environment for documentation builds. Also adds other requirements needed for final docs images.') + +# Supported Python versions for maintained branches. +_python_versions = ['3.6.10', '3.7.7', '3.8.2'] +parser.add_argument('--venvs', nargs='*', type=str, default=_python_versions, + help='List of Python versions ("major.minor.patch") for which to install venvs. ' + 'Default: {}'.format(' '.join(_python_versions))) + + +def image_name(configuration: argparse.Namespace) -> str: + """Generate docker image name. + + The configuration slug has the form:: + + ---[--][-] + + Image name is prefixed by ``gromacs/ci-`` + + Arguments: + configuration: Docker image configuration as described by the parsed arguments. + + """ + elements = [] + for distro in ('centos', 'ubuntu'): + version = getattr(configuration, distro, None) + if version is not None: + elements.append(distro + '-' + version) + break + for compiler in ('icc', 'llvm', 'gcc'): + version = getattr(configuration, compiler, None) + if version is not None: + elements.append(compiler + '-' + str(version).split('.')[0]) + break + for gpusdk in ('cuda',): + version = getattr(configuration, gpusdk, None) + if version is not None: + elements.append(gpusdk + '-' + version) + if configuration.oneapi is not None: + elements.append('oneapi-' + configuration.oneapi) + + # Check for special cases + # The following attribute keys indicate the image is built for the named + # special use case. + cases = {'doxygen': 'docs', + 'tsan': 'tsan'} + for attr in cases: + value = getattr(configuration, attr, None) + if value is not None: + elements.append(cases[attr]) + slug = '-'.join(elements) + return 'gromacs/ci-' + slug + + +if __name__ == "__main__": + args = argparse.ArgumentParser(parents=[parser]).parse_args() + print(image_name(args)) diff --git a/docs/dev-manual/containers.rst b/docs/dev-manual/containers.rst index 82b3c9faa1..20cf4d9c25 100644 --- a/docs/dev-manual/containers.rst +++ b/docs/dev-manual/containers.rst @@ -11,18 +11,19 @@ under :file:`admin/containers/` Images are (re)built manually by |Gromacs| project staff and pushed to repositories at https://hub.docker.com/u/gromacs +Refer to :file:`buildall.sh` in the ``master`` branch for the set of images +currently being built. + Utilities ========= +:file:`utility.py` +------------------ + .. automodule:: utility :members: -HPC container maker -------------------- - -We use the `NVidia HPC Container Maker `__ -package for scripted Dockerfile generation. -See :file:`admin/containers/scripted_gmx_docker_builds.py`. +:file:`scripted_gmx_docker_builds.py` +------------------------------------- -.. todo:: :issue:`3272` Insert tool documentation. - E.g. ``.. automodule:: scripted_gmx_docker_builds`` +.. automodule:: scripted_gmx_docker_builds diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst index a7d5254f18..e705675eb2 100644 --- a/docs/dev-manual/gitlab.rst +++ b/docs/dev-manual/gitlab.rst @@ -1,5 +1,5 @@ -GitLab -====== +GitLab CI Pipeline Execution +============================ The repository contains DockerFiles and GitLab Runner configuration files to support automated testing and documentation builds. @@ -18,12 +18,11 @@ This documentation is incomplete, pending resolution of :issue:`3275`. .. todo:: Expand this documentation to resolve :issue:`3275` -Pipeline execution ------------------- - .. todo:: Discuss the distinct characteristics of |Gromacs| CI pipelines to relevant to job configuration. + (:issue:`3472` and :issue:`3617`) -.. todo:: Comment on the number of pipelines that can be or which are likely to be running at the same time. +.. todo:: (:issue:`3472` and :issue:`3617`) Comment on the number of pipelines that can be or which are likely to be running at the same time. + (:issue:`3472` and :issue:`3617`) .. note:: @@ -35,7 +34,7 @@ Pipeline execution sufficient testing before acceptance. Configuration files -~~~~~~~~~~~~~~~~~~~ +------------------- At the root of the repository, :file:`.gitlab-ci.yml` defines the stages and some default parameters, then includes files from :file:`admin/gitlab-ci/` to @@ -47,7 +46,7 @@ Such jobs are not directly eligible to run, but may be used as templates via the `*extends* job property `_. Job parameters -~~~~~~~~~~~~~~ +-------------- Refer to https://docs.gitlab.com/ee/ci/yaml for complete documentation on GitLab CI job parameters, but note the following GROMACS-specific conventions. @@ -69,9 +68,9 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions. to `cache:key `__ image - Part of the tool chain configuration. Instead of setting *image* - directly, *extend* a *.use_* template from - :file:`admin/gitlab-ci/global.gitlab-ci.yml` + See :doc:`/dev-manual/containers` for more about the Docker images used for the + CI pipelines. If a job depends on artifacts from previous jobs, be sure + to use the same (or a compatible) image as the dependency! rules only @@ -110,7 +109,7 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions. for details of the merging behavior. Refer to :ref:`variables` for local usage. Schedules and triggers -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- Pipeline `schedules `__ are configured through the GitLab web interface. @@ -131,7 +130,7 @@ or one of the *release* branches. Those jobs can be triggered manually using the through the Gitlab web interface. Global templates -~~~~~~~~~~~~~~~~ +---------------- In addition to the templates in the main job definition files, common "mix-in" functionality and behavioral templates are defined in @@ -149,7 +148,7 @@ by a meaningful descriptor and documented within :file:`admin/gitlab-ci/global.gitlab-ci.yml` Job names -~~~~~~~~~ +--------- Job names should @@ -170,7 +169,7 @@ basic job name from qualifiers or details. Also consider .. _variables: Updating regression tests -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- Changes in |Gromacs| that require changes in regression-tests are notoriously hard, because a merge request that tests against the non-updated version of the @@ -181,11 +180,11 @@ merge request pipelines to fail. The solution is a new regression-test branch or commit, uploaded to gitlab. Then set that regression test branch with REGRESSIONTESTBRANCH or the specific commit with REGRESSIONTESTCOMMIT when -running the specific pipeline that requires the regressiontest-update. +running the specific pipeline that requires the regressiontest-update. See below on how to set variables for specific pipelines. Variables -~~~~~~~~~ +--------- The GitLab CI framework, GitLab Runner, plugins, and our own scripts set and use several `variables `__. @@ -238,12 +237,12 @@ Other important variable keys are as follows. pipeline execution time. REGRESSIONTESTBRANCH - Use this branch of the regressiontests rather than master to allow for + Use this branch of the regressiontests rather than master to allow for merge requests that require updated regression tests with valid CI tests. REGRESSIONTESTCOMMIT - Use this commit to the regressiontests rather than the head on master to - allow for merge requests that require updated regression tests with + Use this commit to the regressiontests rather than the head on master to + allow for merge requests that require updated regression tests with valid CI tests. POST_MERGE_ACCEPTANCE @@ -257,7 +256,7 @@ Other important variable keys are as follows. ``BUILD_DIR``, ``INSTALL_DIR``, ``CACHE_FALLBACK_KEY``, ... Setting variables -~~~~~~~~~~~~~~~~~ +----------------- Variables for individual piplelines are set in the gitlab interface under ``CI/CD``; ``Pipelines``. Then chose in the top right corner ``Run Piplelines``. -- 2.22.0