3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2020, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
36 """Building block based Dockerfile generation for CI testing images.
38 Generates a set of docker images used for running GROMACS CI on Gitlab.
39 The images are prepared according to a selection of build configuration targets
40 that hope to cover a broad enough scope of different possible systems,
41 allowing us to check compiler types and versions, as well as libraries used
42 for accelerators and parallel communication systems. Each combinations is
43 described as an entry in the build_configs dictionary, with the script
44 analysing the logic and adding build stages as needed.
46 Based on the example script provided by the NVidia HPCCM repository.
49 `NVidia HPC Container Maker <https://github.com/NVIDIA/hpc-container-maker>`__
52 * Paul Bauer <paul.bauer.q@gmail.com>
53 * Eric Irrgang <ericirrgang@gmail.com>
54 * Joe Jordan <e.jjordan12@gmail.com>
55 * Mark Abraham <mark.j.abraham@gmail.com>
59 $ python3 scripted_gmx_docker_builds.py --help
60 $ python3 scripted_gmx_docker_builds.py --format docker > Dockerfile && docker build .
61 $ python3 scripted_gmx_docker_builds.py | docker build -
71 from distutils.version import StrictVersion
75 from hpccm.building_blocks.base import bb_base
81 'This module assumes availability of supporting modules in the same directory. Add the directory to '
82 'PYTHONPATH or invoke Python from within the module directory so module location can be resolved.')
84 # Basic packages for all final images.
85 _common_packages = ['build-essential',
103 _opencl_extra_packages = [
105 # The following require apt_ppas=['ppa:intel-opencl/intel-opencl']
107 'ocl-icd-libopencl1',
108 'ocl-icd-opencl-dev',
110 # The following require
111 # apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'],
112 # apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main']
119 # Extra packages needed to build Python installations from source.
120 _python_extra_packages = ['build-essential',
139 # Extra packages needed for images for building documentation.
140 _docs_extra_packages = ['autoconf',
156 'texlive-latex-base',
157 'texlive-latex-extra',
158 'texlive-fonts-recommended',
159 'texlive-fonts-extra']
161 # Parse command line arguments
162 parser = argparse.ArgumentParser(description='GROMACS CI image creation script',
163 parents=[utility.parser])
165 parser.add_argument('--format', type=str, default='docker',
166 choices=['docker', 'singularity'],
167 help='Container specification format (default: docker)')
170 def base_image_tag(args) -> str:
171 # Check if we use CUDA images or plain linux images
172 if args.cuda is not None:
173 cuda_version_tag = 'nvidia/cuda:' + args.cuda + '-devel'
174 if args.centos is not None:
175 cuda_version_tag += '-centos' + args.centos
176 elif args.ubuntu is not None:
177 cuda_version_tag += '-ubuntu' + args.ubuntu
179 raise RuntimeError('Logic error: no Linux distribution selected.')
181 base_image_tag = cuda_version_tag
183 if args.centos is not None:
184 base_image_tag = 'centos:centos' + args.centos
185 elif args.ubuntu is not None:
186 base_image_tag = 'ubuntu:' + args.ubuntu
188 raise RuntimeError('Logic error: no Linux distribution selected.')
189 return base_image_tag
192 def get_llvm_packages(args) -> typing.Iterable[str]:
193 # If we use the package version of LLVM, we need to install extra packages for it.
194 if (args.llvm is not None) and (args.tsan is None):
195 return ['libomp-dev',
197 'clang-format-' + str(args.llvm),
198 'clang-tidy-' + str(args.llvm)]
202 def get_opencl_packages(args) -> typing.Iterable[str]:
203 if (args.doxygen is None) and (args.oneapi is None):
204 return _opencl_extra_packages
208 def get_compiler(args, compiler_build_stage: hpccm.Stage = None) -> bb_base:
210 if args.icc is not None:
211 raise RuntimeError('Intel compiler toolchain recipe not implemented yet')
213 if args.llvm is not None:
214 # Build our own version instead to get TSAN + OMP
215 if args.tsan is not None:
216 if compiler_build_stage is not None:
217 compiler = compiler_build_stage.runtime(_from='tsan')
219 raise RuntimeError('No TSAN compiler build stage!')
220 # Build the default compiler if we don't need special support
222 compiler = hpccm.building_blocks.llvm(extra_repository=True, version=args.llvm)
224 elif args.oneapi is not None:
225 if compiler_build_stage is not None:
226 compiler = compiler_build_stage.runtime(_from='oneapi')
227 # Prepare the toolchain (needed only for builds done within the Dockerfile, e.g.
228 # OpenMPI builds, which don't currently work for other reasons)
229 oneapi_toolchain = hpccm.toolchain(CC='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc',
230 CXX='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icpc')
231 setattr(compiler, 'toolchain', oneapi_toolchain)
234 raise RuntimeError('No oneAPI compiler build stage!')
236 elif args.gcc is not None:
237 compiler = hpccm.building_blocks.gnu(extra_repository=True,
241 raise RuntimeError('Logic error: no compiler toolchain selected.')
245 def get_mpi(args, compiler):
246 # If needed, add MPI to the image
247 if args.mpi is not None:
248 if args.mpi == 'openmpi':
250 if args.cuda is not None:
253 if hasattr(compiler, 'toolchain'):
254 if args.oneapi is not None:
255 raise RuntimeError('oneAPI building OpenMPI is not supported')
256 return hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, cuda=use_cuda, infiniband=False)
258 raise RuntimeError('compiler is not an HPCCM compiler building block!')
260 elif args.mpi == 'impi':
261 # TODO Intel MPI from the oneAPI repo is not working reliably,
262 # reasons are unclear. When solved, add packagages called:
263 # 'intel-oneapi-mpi', 'intel-oneapi-mpi-devel'
264 # during the compiler stage.
265 # TODO also consider hpccm's intel_mpi package if that doesn't need
267 raise RuntimeError('Intel MPI recipe not implemented yet.')
269 raise RuntimeError('Requested unknown MPI implementation.')
275 if (args.clfft is not None):
276 return hpccm.building_blocks.generic_cmake(
277 repository='https://github.com/clMathLibraries/clFFT.git',
278 prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src')
283 def add_tsan_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
284 """Isolate the expensive TSAN preparation stage.
286 This is a very expensive stage, but has few and disjoint dependencies, and
287 its output is easily compartmentalized (/usr/local) so we can isolate this
288 build stage to maximize build cache hits and reduce rebuild time, bookkeeping,
289 and final image size.
291 if not isinstance(output_stages, collections.abc.MutableMapping):
292 raise RuntimeError('Need output_stages container.')
293 tsan_stage = hpccm.Stage()
294 tsan_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='tsan')
296 tsan_stage += hpccm.building_blocks.packages(ospackages=['git', 'ca-certificates', 'build-essential', 'cmake'])
297 # CMake will get duplicated later, but this is an expensive image, and it isn't worth optimizing
298 # out that duplication...
299 tsan_stage += hpccm.building_blocks.python(python3=True, python2=False, devel=False)
301 compiler_branch = 'release_' + str(input_args.llvm) + '0'
302 tsan_stage += hpccm.building_blocks.generic_cmake(
303 repository='https://git.llvm.org/git/llvm.git',
304 prefix='/usr/local', recursive=True, branch=compiler_branch,
305 cmake_opts=['-D CMAKE_BUILD_TYPE=Release', '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra"',
306 '-D LIBOMP_TSAN_SUPPORT=on'],
307 preconfigure=['export branch=' + compiler_branch,
308 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxx.git)',
309 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxxabi.git)',
310 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/compiler-rt.git)',
311 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/openmp.git)',
312 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang.git)',
313 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang-tools-extra.git)'],
314 postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-' + str(input_args.llvm),
315 'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-' + str(input_args.llvm),
316 'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-' + str(input_args.llvm),
317 'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-' + str(input_args.llvm)])
318 output_stages['compiler_build'] = tsan_stage
320 def oneapi_runtime(_from='0'):
321 oneapi_runtime_stage = hpccm.Stage()
322 oneapi_runtime_stage += hpccm.primitives.copy(_from='oneapi-build',
323 files={"/opt/intel": "/opt/intel",
324 "/etc/bash.bashrc": "/etc/bash.bashrc"})
325 return oneapi_runtime_stage
327 def add_oneapi_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
328 """Isolate the oneAPI preparation stage.
330 This stage is isolated so that its installed components are minimized in the
331 final image (chiefly /opt/intel) and its environment setup script can be
332 sourced. This also helps with rebuild time and final image size.
334 Note that the ICC compiler inside oneAPI on linux also needs
335 gcc to build other components and provide libstdc++.
337 if not isinstance(output_stages, collections.abc.MutableMapping):
338 raise RuntimeError('Need output_stages container.')
339 oneapi_stage = hpccm.Stage()
340 oneapi_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='oneapi-build')
342 version = str(input_args.oneapi)
344 # Add required components for the next stage (both for hpccm and Intel's setvars.sh script)
345 oneapi_stage += hpccm.building_blocks.packages(ospackages=['wget', 'gnupg2', 'ca-certificates', 'lsb-release'])
346 oneapi_stage += hpccm.building_blocks.packages(
347 apt_keys=['https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'],
348 apt_repositories=['deb https://apt.repos.intel.com/oneapi all main'],
349 # Add minimal packages (not the whole HPC toolkit!)
350 ospackages=['intel-oneapi-dpcpp-cpp-compiler-pro-{}'.format(version),
351 'intel-oneapi-openmp-{}'.format(version),
352 'intel-oneapi-mkl-{}'.format(version),
353 'intel-oneapi-mkl-devel-{}'.format(version)]
355 # Ensure that all bash shells on the final container will have access to oneAPI
356 oneapi_stage += hpccm.primitives.shell(
357 commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc']
359 setattr(oneapi_stage, 'runtime', oneapi_runtime)
361 output_stages['compiler_build'] = oneapi_stage
363 def prepare_venv(version: StrictVersion) -> typing.Sequence[str]:
364 """Get shell commands to set up the venv for the requested Python version."""
365 major = version.version[0]
366 minor = version.version[1] # type: int
368 pyenv = '$HOME/.pyenv/bin/pyenv'
370 py_ver = '{}.{}'.format(major, minor)
371 venv_path = '$HOME/venv/py{}'.format(py_ver)
372 commands = ['$({pyenv} prefix `{pyenv} whence python{py_ver}`)/bin/python -m venv {path}'.format(
378 commands.append('{path}/bin/python -m pip install --upgrade pip setuptools'.format(
381 # Install dependencies for building and testing gmxapi Python package.
382 # WARNING: Please keep this list synchronized with python_packaging/requirements-test.txt
383 # TODO: Get requirements.txt from an input argument.
384 commands.append("""{path}/bin/python -m pip install --upgrade \
393 'scikit-build>=0.10'""".format(path=venv_path))
395 # TODO: Remove 'importlib_resources' dependency when Python >=3.7 is required.
397 commands.append("""{path}/bin/python -m pip install --upgrade \
398 'importlib_resources'""".format(path=venv_path))
403 def add_python_stages(building_blocks: typing.Mapping[str, bb_base],
405 output_stages: typing.MutableMapping[str, hpccm.Stage]):
406 """Add the stage(s) necessary for the requested venvs.
408 One intermediate build stage is created for each venv (see --venv option).
410 Each stage partially populates Python installations and venvs in the home
411 directory. The home directory is collected by the 'pyenv' stage for use by
412 the main build stage.
414 if len(input_args.venvs) < 1:
415 raise RuntimeError('No venvs to build...')
416 if output_stages is None or not isinstance(output_stages, collections.abc.Mapping):
417 raise RuntimeError('Need a container for output stages.')
419 # Main Python stage that collects the environments from individual stages.
420 # We collect the stages individually, rather than chaining them, because the
421 # copy is a bit slow and wastes local Docker image space for each filesystem
423 pyenv_stage = hpccm.Stage()
424 pyenv_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='pyenv')
425 pyenv_stage += building_blocks['compiler']
426 pyenv_stage += building_blocks['mpi']
427 pyenv_stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
429 for version in [StrictVersion(py_ver) for py_ver in sorted(input_args.venvs)]:
430 stage_name = 'py' + str(version)
431 stage = hpccm.Stage()
432 stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as=stage_name)
433 stage += building_blocks['compiler']
434 stage += building_blocks['mpi']
435 stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
437 # TODO: Use a non-root user for testing and Python virtual environments.
438 stage += hpccm.primitives.shell(commands=[
439 'curl https://pyenv.run | bash',
440 """echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $HOME/.bashrc""",
441 """echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> $HOME/.bashrc""",
442 """echo 'eval "$(pyenv init -)"' >> $HOME/.bashrc""",
443 """echo 'eval "$(pyenv virtualenv-init -)"' >> $HOME/.bashrc"""])
444 pyenv = '$HOME/.pyenv/bin/pyenv'
445 commands = ['PYTHON_CONFIGURE_OPTS="--enable-shared" {pyenv} install -s {version}'.format(
447 version=str(version))]
448 stage += hpccm.primitives.shell(commands=commands)
450 commands = prepare_venv(version)
451 stage += hpccm.primitives.shell(commands=commands)
453 # TODO: Update user home directory.
454 pyenv_stage += hpccm.primitives.copy(_from=stage_name, _mkdir=True, src=['/root/'],
457 # Add the intermediate build stage to the sequence
458 output_stages[stage_name] = stage
460 # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
461 # # `version` should be a system installation or pyenv environment (or pyenv-virtualenv)
462 # # with the dependencies for all of the Python aspects of CMake-driven builds.
463 # commands = '{pyenv} global {version}'.format(
466 # pyenv_stage += hpccm.primitives.shell(commands=commands)
468 # Add the aggregating build stage to the sequence. This allows the main stage to copy
469 # the files in a single stage, potentially reducing the overall output image size.
470 output_stages['pyenv'] = pyenv_stage
473 def add_documentation_dependencies(input_args,
474 output_stages: typing.MutableMapping[str, hpccm.Stage]):
475 """Add appropriate layers according to doxygen input arguments."""
476 if input_args.doxygen is None:
478 output_stages['main'] += hpccm.primitives.shell(
479 commands=['sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml'])
480 output_stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1', 'gcovr'])
481 if input_args.doxygen == '1.8.5':
482 doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a'
483 output_stages['main'] += hpccm.building_blocks.generic_autotools(
484 repository='https://github.com/westes/flex.git',
485 commit='f7788a9a0ecccdc953ed12043ccb59ca25714018',
486 prefix='/tmp/install-of-flex',
487 configure_opts=['--disable-shared'],
488 preconfigure=['./autogen.sh'])
489 output_stages['main'] += hpccm.building_blocks.generic_autotools(
490 repository='https://github.com/doxygen/doxygen.git',
491 commit=doxygen_commit,
494 '--flex /tmp/install-of-flex/bin/flex',
497 version = input_args.doxygen
498 archive_name = 'doxygen-{}.linux.bin.tar.gz'.format(version)
499 archive_url = 'https://sourceforge.net/projects/doxygen/files/rel-{}/{}'.format(
503 binary_path = 'doxygen-{}/bin/doxygen'.format(version)
505 'mkdir doxygen && cd doxygen',
506 'wget {}'.format(archive_url),
507 'tar xf {} {}'.format(archive_name, binary_path),
508 'cp {} /usr/local/bin/'.format(binary_path),
509 'cd .. && rm -rf doxygen'
511 output_stages['main'] += hpccm.primitives.shell(commands=commands)
514 def build_stages(args) -> typing.Iterable[hpccm.Stage]:
515 """Define and sequence the stages for the recipe corresponding to *args*."""
517 # A Dockerfile or Singularity recipe can have multiple build stages.
518 # The main build stage can copy files from previous stages, though only
519 # the last stage is included in the tagged output image. This means that
520 # large or expensive sets of build instructions can be isolated in
521 # local/temporary images, but all of the stages need to be output by this
522 # script, and need to occur in the correct order, so we create a sequence
523 # object early in this function.
524 stages = collections.OrderedDict()
526 # If we need TSAN or oneAPI support the early build is more complex,
527 # so that our compiler images don't have all the cruft needed to get those things
529 if args.llvm is not None and args.tsan is not None:
530 add_tsan_compiler_build_stage(input_args=args, output_stages=stages)
531 if args.oneapi is not None:
532 add_oneapi_compiler_build_stage(input_args=args, output_stages=stages)
534 # Building blocks are chunks of container-builder instructions that can be
535 # copied to any build stage with the addition operator.
536 building_blocks = collections.OrderedDict()
537 building_blocks['base_packages'] = hpccm.building_blocks.packages(
538 ospackages=_common_packages)
540 # These are the most expensive and most reusable layers, so we put them first.
541 building_blocks['compiler'] = get_compiler(args, compiler_build_stage=stages.get('compiler_build'))
542 building_blocks['mpi'] = get_mpi(args, building_blocks['compiler'])
543 for i, cmake in enumerate(args.cmake):
544 building_blocks['cmake' + str(i)] = hpccm.building_blocks.cmake(
546 prefix='/usr/local/cmake-{}'.format(cmake),
549 # Install additional packages early in the build to optimize Docker build layer cache.
550 os_packages = list(get_llvm_packages(args)) + get_opencl_packages(args)
551 if args.doxygen is not None:
552 os_packages += _docs_extra_packages
553 if args.oneapi is not None:
554 os_packages += ['lsb-release']
555 building_blocks['extra_packages'] = hpccm.building_blocks.packages(
556 ospackages=os_packages,
557 apt_ppas=['ppa:intel-opencl/intel-opencl'],
558 apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'],
559 apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main']
562 building_blocks['clfft'] = get_clfft(args)
564 # Add Python environments to MPI images, only, so we don't have to worry
565 # about whether to install mpi4py.
566 if args.mpi is not None and len(args.venvs) > 0:
567 add_python_stages(building_blocks=building_blocks, input_args=args, output_stages=stages)
569 # Create the stage from which the targeted image will be tagged.
570 stages['main'] = hpccm.Stage()
572 stages['main'] += hpccm.primitives.baseimage(image=base_image_tag(args))
573 for bb in building_blocks.values():
577 # We always add Python3 and Pip
578 stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True)
579 stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3',
580 packages=['pytest', 'networkx', 'numpy'])
582 # Add documentation requirements (doxygen and sphinx + misc).
583 if args.doxygen is not None:
584 add_documentation_dependencies(args, stages)
586 if 'pyenv' in stages and stages['pyenv'] is not None:
587 stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/.pyenv/'],
589 stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/venv/'],
591 # TODO: Update user home directory.
592 # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
593 # stages['main'] += hpccm.primitives.copy(_from='pyenv', src=['/root/.bashrc'],
596 # Make sure that `python` resolves to something.
597 stages['main'] += hpccm.primitives.shell(commands=['test -x /usr/bin/python || '
598 'update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && '
599 '/usr/bin/python --version'])
601 # Note that the list of stages should be sorted in dependency order.
602 for build_stage in stages.values():
603 if build_stage is not None:
607 if __name__ == '__main__':
608 args = parser.parse_args()
610 # Set container specification output format
611 hpccm.config.set_container_format(args.format)
613 container_recipe = build_stages(args)
615 # Output container specification
616 for stage in container_recipe: