admin/containers/scripted_gmx_docker_builds.py

   1 #!/usr/bin/env python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2020,2021, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """Building block based Dockerfile generation for CI testing images.
  37
  38 Generates a set of docker images used for running GROMACS CI on Gitlab.
  39 The images are prepared according to a selection of build configuration targets
  40 that hope to cover a broad enough scope of different possible systems,
  41 allowing us to check compiler types and versions, as well as libraries used
  42 for accelerators and parallel communication systems. Each combinations is
  43 described as an entry in the build_configs dictionary, with the script
  44 analysing the logic and adding build stages as needed.
  45
  46 Based on the example script provided by the NVidia HPCCM repository.
  47
  48 Reference:
  49     `NVidia HPC Container Maker <https://github.com/NVIDIA/hpc-container-maker>`__
  50
  51 Authors:
  52     * Paul Bauer <paul.bauer.q@gmail.com>
  53     * Eric Irrgang <ericirrgang@gmail.com>
  54     * Joe Jordan <e.jjordan12@gmail.com>
  55     * Mark Abraham <mark.j.abraham@gmail.com>
  56     * Gaurav Garg <gaugarg@nvidia.com>
  57
  58 Usage::
  59
  60     $ python3 scripted_gmx_docker_builds.py --help
  61     $ python3 scripted_gmx_docker_builds.py --format docker > Dockerfile && docker build .
  62     $ python3 scripted_gmx_docker_builds.py | docker build -
  63
  64 See Also:
  65     :file:`buildall.sh`
  66
  67 """
  68
  69 import argparse
  70 import collections
  71 import collections.abc
  72 import typing
  73 from distutils.version import StrictVersion
  74
  75 import hpccm
  76 import hpccm.config
  77 from hpccm.building_blocks.base import bb_base
  78
  79 try:
  80     import utility
  81 except ImportError:
  82     raise RuntimeError(
  83         'This module assumes availability of supporting modules in the same directory. Add the directory to '
  84         'PYTHONPATH or invoke Python from within the module directory so module location can be resolved.')
  85
  86 # Basic packages for all final images.
  87 _common_packages = ['build-essential',
  88                     'ca-certificates',
  89                     'ccache',
  90                     'git',
  91                     'gnupg',
  92                     'gpg-agent',
  93                     'libfftw3-dev',
  94                     'libhwloc-dev',
  95                     'liblapack-dev',
  96                     'libx11-dev',
  97                     'moreutils',
  98                     'ninja-build',
  99                     'rsync',
 100                     'valgrind',
 101                     'vim',
 102                     'wget',
 103                     'xsltproc']
 104
 105 _opencl_extra_packages = [
 106     'nvidia-opencl-dev',
 107     # The following require apt_ppas=['ppa:intel-opencl/intel-opencl']
 108     'intel-opencl-icd',
 109     'ocl-icd-libopencl1',
 110     'ocl-icd-opencl-dev',
 111     'opencl-headers',
 112 ]
 113
 114 _rocm_extra_packages = [
 115     # The following require
 116     #             apt_keys=['http://repo.radeon.com/rocm/rocm.gpg.key'],
 117     #             apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/4.0.1/ xenial main']
 118     'clinfo',
 119     'hipfft',
 120     'libelf1',
 121     'rocfft',
 122     'rocm-opencl',
 123     'rocm-dev',
 124 ]
 125
 126 # Extra packages needed to build Intel Compute Runtime
 127 _intel_compute_runtime_extra_packages = ['libigc',
 128                                          'libigc-dev',
 129                                          'libigdgmm11',
 130                                          'libigdgmm-dev',
 131                                          'libxml2',
 132                                          'libxml2-dev',
 133                                          'libze-loader',
 134                                          'libze-loader-dev',
 135                                          'ocl-icd-libopencl1',
 136                                          'ocl-icd-opencl-dev',
 137                                          'opencl-headers',
 138                                          'pkg-config',
 139                                          ]
 140
 141 # Extra packages needed to build Python installations from source.
 142 _python_extra_packages = ['build-essential',
 143                           'ca-certificates',
 144                           'ccache',
 145                           'curl',
 146                           'git',
 147                           'libbz2-dev',
 148                           'libffi-dev',
 149                           'liblzma-dev',
 150                           'libncurses5-dev',
 151                           'libncursesw5-dev',
 152                           'libreadline-dev',
 153                           'libsqlite3-dev',
 154                           'libssl-dev',
 155                           'llvm',
 156                           'python-openssl',
 157                           'vim',
 158                           'wget',
 159                           'zlib1g-dev']
 160
 161 # Extra packages needed for images for building documentation.
 162 _docs_extra_packages = ['autoconf',
 163                         'automake',
 164                         'autopoint',
 165                         'autotools-dev',
 166                         'bison',
 167                         'flex',
 168                         'ghostscript',
 169                         'graphviz',
 170                         'help2man',
 171                         'imagemagick',
 172                         'libtool',
 173                         'mscgen',
 174                         'm4',
 175                         'openssh-client',
 176                         'texinfo',
 177                         'texlive-latex-base',
 178                         'texlive-latex-extra',
 179                         'texlive-fonts-recommended',
 180                         'texlive-fonts-extra',
 181                         'tex-gyre']
 182
 183 # Parse command line arguments
 184 parser = argparse.ArgumentParser(description='GROMACS CI image creation script',
 185                                  parents=[utility.parser])
 186
 187 parser.add_argument('--format', type=str, default='docker',
 188                     choices=['docker', 'singularity'],
 189                     help='Container specification format (default: docker)')
 190
 191
 192 def base_image_tag(args) -> str:
 193     """Generate *image* for hpccm.baseimage()."""
 194     # Check if we use CUDA images or plain linux images
 195     if args.cuda is not None:
 196         cuda_version_tag = 'nvidia/cuda:' + args.cuda + '-devel'
 197         if args.centos is not None:
 198             cuda_version_tag += '-centos' + args.centos
 199         elif args.ubuntu is not None:
 200             cuda_version_tag += '-ubuntu' + args.ubuntu
 201         else:
 202             raise RuntimeError('Logic error: no Linux distribution selected.')
 203
 204         base_image_tag = cuda_version_tag
 205     else:
 206         if args.centos is not None:
 207             base_image_tag = 'centos:centos' + args.centos
 208         elif args.ubuntu is not None:
 209             base_image_tag = 'ubuntu:' + args.ubuntu
 210         else:
 211             raise RuntimeError('Logic error: no Linux distribution selected.')
 212     return base_image_tag
 213
 214
 215 def hpccm_distro_name(args) -> str:
 216     """Generate *_distro* for hpccm.baseimage().
 217
 218     Convert the linux distribution variables into something that hpccm
 219     understands.
 220
 221     The same format is used by the lower level hpccm.config.set_linux_distro().
 222     """
 223     if args.centos is not None:
 224         name_mapping = {'7': 'centos7',
 225                         '8': 'centos8'}
 226         if args.centos in name_mapping:
 227             hpccm_name = name_mapping[args.centos]
 228         else:
 229             raise RuntimeError('Logic error: unsupported CentOS distribution selected.')
 230     elif args.ubuntu is not None:
 231         name_mapping = {'20.04': 'ubuntu20',
 232                         '18.04': 'ubuntu18',
 233                         '16.04': 'ubuntu16'}
 234         if args.ubuntu in name_mapping:
 235             hpccm_name = name_mapping[args.ubuntu]
 236         else:
 237             raise RuntimeError('Logic error: unsupported Ubuntu distribution selected.')
 238     else:
 239         raise RuntimeError('Logic error: no Linux distribution selected.')
 240     return hpccm_name
 241
 242
 243 def get_llvm_packages(args) -> typing.Iterable[str]:
 244     # If we use the package version of LLVM, we need to install extra packages for it.
 245     if (args.llvm is not None) and (args.tsan is None):
 246         packages = [f'libomp-{args.llvm}-dev',
 247                     f'libomp5-{args.llvm}',
 248                     'clang-format-' + str(args.llvm),
 249                     'clang-tidy-' + str(args.llvm)]
 250         if args.hipsycl is not None:
 251             packages += [f'llvm-{args.llvm}-dev',
 252                          f'libclang-{args.llvm}-dev',
 253                          f'lld-{args.llvm}']
 254         return packages
 255     else:
 256         return []
 257
 258
 259 def get_opencl_packages(args) -> typing.List[str]:
 260     if (args.doxygen is None) and (args.oneapi is None):
 261         return _opencl_extra_packages
 262     else:
 263         return []
 264
 265
 266 def get_rocm_packages(args) -> typing.List[str]:
 267     if (args.rocm is None):
 268         return []
 269     else:
 270         return _rocm_extra_packages
 271
 272
 273 def get_compiler(args, compiler_build_stage: hpccm.Stage = None) -> bb_base:
 274     # Compiler
 275     if args.llvm is not None:
 276         # Build our own version instead to get TSAN + OMP
 277         if args.tsan is not None:
 278             if compiler_build_stage is not None:
 279                 compiler = compiler_build_stage.runtime(_from='tsan')
 280             else:
 281                 raise RuntimeError('No TSAN compiler build stage!')
 282         # Build the default compiler if we don't need special support
 283         else:
 284             # Currently the focal apt repositories do not contain
 285             # llvm higher than 11, so we work around that. This will
 286             # need further work when we start supporting ubuntu 22.04
 287             compiler = hpccm.building_blocks.llvm(version=args.llvm, upstream=True if int(args.llvm) > 11 else False)
 288
 289     elif args.oneapi is not None:
 290         if compiler_build_stage is not None:
 291             compiler = compiler_build_stage.runtime(_from='oneapi')
 292             # Prepare the toolchain (needed only for builds done within the Dockerfile, e.g.
 293             # OpenMPI builds, which don't currently work for other reasons)
 294             oneapi_toolchain = hpccm.toolchain(CC=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icx',
 295                                                CXX=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icpx')
 296             setattr(compiler, 'toolchain', oneapi_toolchain)
 297
 298         else:
 299             raise RuntimeError('No oneAPI compiler build stage!')
 300
 301     elif args.gcc is not None:
 302         compiler = hpccm.building_blocks.gnu(extra_repository=True,
 303                                              version=args.gcc,
 304                                              fortran=False)
 305     else:
 306         raise RuntimeError('Logic error: no compiler toolchain selected.')
 307     return compiler
 308
 309
 310 def get_gdrcopy(args, compiler):
 311     if args.cuda is not None:
 312         if hasattr(compiler, 'toolchain'):
 313             # Version last updated June 7, 2021
 314             return hpccm.building_blocks.gdrcopy(toolchain=compiler.toolchain, version="2.2")
 315         else:
 316             raise RuntimeError('compiler is not an HPCCM compiler building block!')
 317     else:
 318         return None
 319
 320
 321 def get_ucx(args, compiler, gdrcopy):
 322     if args.cuda is not None:
 323         if hasattr(compiler, 'toolchain'):
 324             use_gdrcopy = (gdrcopy is not None)
 325             # Version last updated June 7, 2021
 326             return hpccm.building_blocks.ucx(toolchain=compiler.toolchain, gdrcopy=use_gdrcopy, version="1.10.1",
 327                                              cuda=True)
 328         else:
 329             raise RuntimeError('compiler is not an HPCCM compiler building block!')
 330     else:
 331         return None
 332
 333
 334 def get_mpi(args, compiler, ucx):
 335     # If needed, add MPI to the image
 336     if args.mpi is not None:
 337         if args.mpi == 'openmpi':
 338             if hasattr(compiler, 'toolchain'):
 339                 if args.oneapi is not None:
 340                     raise RuntimeError('oneAPI building OpenMPI is not supported')
 341                 use_cuda = (args.cuda is not None)
 342                 use_ucx = (ucx is not None)
 343                 # Version last updated June 7, 2021
 344                 return hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, version="4.1.1", cuda=use_cuda,
 345                                                      ucx=use_ucx, infiniband=False)
 346             else:
 347                 raise RuntimeError('compiler is not an HPCCM compiler building block!')
 348
 349         elif args.mpi == 'impi':
 350             # TODO Intel MPI from the oneAPI repo is not working reliably,
 351             # reasons are unclear. When solved, add packagages called:
 352             # 'intel-oneapi-mpi', 'intel-oneapi-mpi-devel'
 353             # during the compiler stage.
 354             # TODO also consider hpccm's intel_mpi package if that doesn't need
 355             # a license to run.
 356             raise RuntimeError('Intel MPI recipe not implemented yet.')
 357         else:
 358             raise RuntimeError('Requested unknown MPI implementation.')
 359     else:
 360         return None
 361
 362
 363 def get_clfft(args):
 364     if (args.clfft is not None):
 365         return hpccm.building_blocks.generic_cmake(
 366             repository='https://github.com/clMathLibraries/clFFT.git',
 367             prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src')
 368     else:
 369         return None
 370
 371
 372 def get_heffte(args):
 373     if (args.heffte is not None):
 374         return hpccm.building_blocks.generic_cmake(
 375             cmake_opts=['-D CMAKE_BUILD_TYPE=Release',
 376                         '-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda',
 377                         '-D Heffte_ENABLE_CUDA=ON',
 378                         '-D Heffte_ENABLE_FFTW=OFF',
 379                         '-D BUILD_SHARED_LIBS=ON'],
 380             repository='https://bitbucket.org/icl/heffte.git',
 381             prefix='/usr/local', recursive=True, commit=args.heffte, directory='heffte')
 382     else:
 383         return None
 384
 385
 386 def get_hipsycl(args):
 387     if args.hipsycl is None:
 388         return None
 389     if args.llvm is None:
 390         raise RuntimeError('Can not build hipSYCL without llvm')
 391
 392     if args.rocm is None:
 393         raise RuntimeError('hipSYCL requires the rocm packages')
 394
 395     cmake_opts = ['-DLLVM_DIR=/opt/rocm/llvm/lib/cmake/llvm',
 396                   '-DCMAKE_PREFIX_PATH=/opt/rocm/lib/cmake',
 397                   '-DWITH_ROCM_BACKEND=ON']
 398     if args.cuda is not None:
 399         cmake_opts += ['-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda',
 400                        '-DWITH_CUDA_BACKEND=ON']
 401
 402     postinstall = [
 403         # https://github.com/illuhad/hipSYCL/issues/361#issuecomment-718943645
 404         'for f in /opt/rocm/amdgcn/bitcode/*.bc; do ln -s "$f" "/opt/rocm/lib/$(basename $f .bc).amdgcn.bc"; done'
 405     ]
 406     if args.cuda is not None:
 407         postinstall += [
 408             # https://github.com/illuhad/hipSYCL/issues/410#issuecomment-743301929
 409             f'sed s/_OPENMP/__OPENMP_NVPTX__/ -i /usr/lib/llvm-{args.llvm}/lib/clang/*/include/__clang_cuda_complex_builtins.h',
 410             # Not needed unless we're building with CUDA 11.x, but no harm in doing always
 411             'ln -s /usr/local/cuda/compat/* /usr/local/cuda/lib64/'
 412         ]
 413
 414     return hpccm.building_blocks.generic_cmake(
 415         repository='https://github.com/illuhad/hipSYCL.git',
 416         directory='/var/tmp/hipSYCL',
 417         prefix='/usr/local', recursive=True, commit=args.hipsycl,
 418         cmake_opts=['-DCMAKE_BUILD_TYPE=Release', *cmake_opts],
 419         postinstall=postinstall)
 420
 421
 422 def get_intel_compute_runtime(args):
 423     # The only reason we need to build Compute Runtime ourselves is because Intel packages have no DG1 support
 424     # Otherwise, we could have just installed DEB packages from GitHub or Intel PPA
 425     if args.intel_compute_runtime is None:
 426         return None
 427
 428     cmake_opts = ['-DCMAKE_BUILD_TYPE=Release',
 429                   '-DSKIP_UNIT_TESTS=TRUE',
 430                   '-DSUPPORT_GEN8=0', '-DSUPPORT_GEN9=1', '-DSUPPORT_GEN11=1', '-DSUPPORT_GEN12LP=1', '-DSUPPORT_DG1=1',
 431                   '-DBUILD_WITH_L0=1']
 432
 433     return hpccm.building_blocks.generic_cmake(
 434         repository='https://github.com/intel/compute-runtime.git',
 435         directory='compute-runtime',
 436         prefix='/usr/local', recursive=True, branch=args.intel_compute_runtime,
 437         cmake_opts=cmake_opts,
 438         postinstall=['ldconfig'])
 439
 440
 441 def add_tsan_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
 442     """Isolate the expensive TSAN preparation stage.
 443
 444     This is a very expensive stage, but has few and disjoint dependencies, and
 445     its output is easily compartmentalized (/usr/local) so we can isolate this
 446     build stage to maximize build cache hits and reduce rebuild time, bookkeeping,
 447     and final image size.
 448     """
 449     if not isinstance(output_stages, collections.abc.MutableMapping):
 450         raise RuntimeError('Need output_stages container.')
 451     if 'compiler_build' in output_stages:
 452         raise RuntimeError('"compiler_build" output stage is already present.')
 453     tsan_stage = hpccm.Stage()
 454     tsan_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args),
 455                                              _distro=hpccm_distro_name(input_args),
 456                                              _as='tsan')
 457
 458     tsan_stage += hpccm.building_blocks.packages(ospackages=['git', 'ca-certificates', 'build-essential', 'cmake'])
 459     # CMake will get duplicated later, but this is an expensive image, and it isn't worth optimizing
 460     # out that duplication...
 461     tsan_stage += hpccm.building_blocks.python(python3=True, python2=False, devel=False)
 462
 463     compiler_branch = 'release/' + str(input_args.llvm) + '.x'
 464     tsan_stage += hpccm.building_blocks.generic_cmake(
 465         repository='https://github.com/llvm/llvm-project.git',
 466         directory='/var/tmp/llvm-project/llvm/',
 467         prefix='/usr/local', recursive=True, branch=compiler_branch,
 468         cmake_opts=['-D CMAKE_BUILD_TYPE=Release',
 469                     '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra;compiler-rt;lld"',
 470                     '-D LIBOMP_TSAN_SUPPORT=on'],
 471         postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-' + str(input_args.llvm),
 472                      'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-' + str(input_args.llvm),
 473                      'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-' + str(input_args.llvm),
 474                      'ln -s /usr/local/share/clang/run-clang-tidy.py /usr/local/bin/run-clang-tidy-'
 475                      + str(input_args.llvm) + '.py',
 476                      'ln -s /usr/local/bin/run-clang-tidy-'
 477                      + str(input_args.llvm) + '.py /usr/local/bin/run-clang-tidy-' + str(input_args.llvm),
 478                      'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-' + str(input_args.llvm)])
 479     output_stages['compiler_build'] = tsan_stage
 480
 481
 482 def oneapi_runtime(_from='0'):
 483     oneapi_runtime_stage = hpccm.Stage()
 484     oneapi_runtime_stage += hpccm.primitives.copy(_from='oneapi-build',
 485                                                   files={"/opt/intel": "/opt/intel",
 486                                                          "/etc/bash.bashrc": "/etc/bash.bashrc"})
 487     return oneapi_runtime_stage
 488
 489
 490 def add_oneapi_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
 491     """Isolate the oneAPI preparation stage.
 492
 493     This stage is isolated so that its installed components are minimized in the
 494     final image (chiefly /opt/intel) and its environment setup script can be
 495     sourced. This also helps with rebuild time and final image size.
 496     """
 497     if not isinstance(output_stages, collections.abc.MutableMapping):
 498         raise RuntimeError('Need output_stages container.')
 499     if 'compiler_build' in output_stages:
 500         raise RuntimeError('"compiler_build" output stage is already present.')
 501     oneapi_stage = hpccm.Stage()
 502     oneapi_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args),
 503                                                _distro=hpccm_distro_name(input_args),
 504                                                _as='oneapi-build')
 505
 506     version = str(input_args.oneapi)
 507
 508     # Add required components for the next stage (both for hpccm and Intel's setvars.sh script)
 509     oneapi_stage += hpccm.building_blocks.packages(ospackages=['wget', 'gnupg2', 'ca-certificates', 'lsb-release'])
 510     oneapi_stage += hpccm.building_blocks.packages(
 511         apt_keys=['https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'],
 512         apt_repositories=['deb https://apt.repos.intel.com/oneapi all main'],
 513         # Add minimal packages (not the whole HPC toolkit!)
 514         ospackages=[f'intel-oneapi-dpcpp-cpp-{version}',
 515                     f'intel-oneapi-openmp-{version}',
 516                     f'intel-oneapi-mkl-{version}',
 517                     f'intel-oneapi-mkl-devel-{version}']
 518     )
 519     # Ensure that all bash shells on the final container will have access to oneAPI
 520     oneapi_stage += hpccm.primitives.shell(
 521         commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc',
 522                   'unlink /opt/intel/oneapi/compiler/latest',
 523                   f'ln -sf /opt/intel/oneapi/compiler/{version} /opt/intel/oneapi/compiler/latest']
 524     )
 525     setattr(oneapi_stage, 'runtime', oneapi_runtime)
 526
 527     output_stages['compiler_build'] = oneapi_stage
 528
 529
 530 def prepare_venv(version: StrictVersion) -> typing.Sequence[str]:
 531     """Get shell commands to set up the venv for the requested Python version."""
 532     major = version.version[0]
 533     minor = version.version[1]  # type: int
 534
 535     pyenv = '$HOME/.pyenv/bin/pyenv'
 536
 537     py_ver = f'{major}.{minor}'
 538     venv_path = f'$HOME/venv/py{py_ver}'
 539     commands = [f'$({pyenv} prefix `{pyenv} whence python{py_ver}`)/bin/python -m venv {venv_path}']
 540
 541     commands.append(f'{venv_path}/bin/python -m pip install --upgrade pip setuptools')
 542     # Install dependencies for building and testing gmxapi Python package.
 543     # WARNING: Please keep this list synchronized with python_packaging/requirements-test.txt
 544     # TODO: Get requirements.txt from an input argument.
 545     commands.append(f"""{venv_path}/bin/python -m pip install --upgrade \
 546             'breathe' \
 547             'cmake>=3.16.3' \
 548             'flake8>=3.7.7' \
 549             'gcovr>=4.2' \
 550             'mpi4py>=3.0.3' \
 551             'networkx>=2.0' \
 552             'numpy>=1' \
 553             'pip>=10.1' \
 554             'pybind11>2.6' \
 555             'Pygments>=2.2.0' \
 556             'pytest>=3.9' \
 557             'setuptools>=42' \
 558             'scikit-build>=0.10' \
 559             'Sphinx>=1.6.3' \
 560             'sphinxcontrib-plantuml>=0.14' \
 561             'wheel'""")
 562     return commands
 563
 564
 565 def add_python_stages(building_blocks: typing.Mapping[str, bb_base],
 566                       input_args,
 567                       output_stages: typing.MutableMapping[str, hpccm.Stage]):
 568     """Add the stage(s) necessary for the requested venvs.
 569
 570     One intermediate build stage is created for each venv (see --venv option).
 571
 572     Each stage partially populates Python installations and venvs in the home
 573     directory. The home directory is collected by the 'pyenv' stage for use by
 574     the main build stage.
 575     """
 576     if len(input_args.venvs) < 1:
 577         raise RuntimeError('No venvs to build...')
 578     if output_stages is None or not isinstance(output_stages, collections.abc.Mapping):
 579         raise RuntimeError('Need a container for output stages.')
 580
 581     # Main Python stage that collects the environments from individual stages.
 582     # We collect the stages individually, rather than chaining them, because the
 583     # copy is a bit slow and wastes local Docker image space for each filesystem
 584     # layer.
 585     pyenv_stage = hpccm.Stage()
 586     pyenv_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args),
 587                                               _distro=hpccm_distro_name(input_args),
 588                                               _as='pyenv')
 589     pyenv_stage += building_blocks['compiler']
 590     if building_blocks['gdrcopy'] is not None:
 591         pyenv_stage += building_blocks['gdrcopy']
 592     if building_blocks['ucx'] is not None:
 593         pyenv_stage += building_blocks['ucx']
 594     pyenv_stage += building_blocks['mpi']
 595     pyenv_stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
 596
 597     for version in [StrictVersion(py_ver) for py_ver in sorted(input_args.venvs)]:
 598         stage_name = 'py' + str(version)
 599         stage = hpccm.Stage()
 600         stage += hpccm.primitives.baseimage(image=base_image_tag(input_args),
 601                                             _distro=hpccm_distro_name(input_args),
 602                                             _as=stage_name)
 603         stage += building_blocks['compiler']
 604         if building_blocks['gdrcopy'] is not None:
 605             stage += building_blocks['gdrcopy']
 606         if building_blocks['ucx'] is not None:
 607             stage += building_blocks['ucx']
 608         stage += building_blocks['mpi']
 609         stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
 610
 611         # TODO: Use a non-root user for testing and Python virtual environments.
 612         stage += hpccm.primitives.shell(commands=[
 613             'curl https://pyenv.run | bash',
 614             """echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $HOME/.bashrc""",
 615             """echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> $HOME/.bashrc""",
 616             """echo 'eval "$(pyenv init -)"' >> $HOME/.bashrc""",
 617             """echo 'eval "$(pyenv virtualenv-init -)"' >> $HOME/.bashrc"""])
 618         pyenv = '$HOME/.pyenv/bin/pyenv'
 619         commands = [f'PYTHON_CONFIGURE_OPTS="--enable-shared" {pyenv} install -s {version}']
 620         stage += hpccm.primitives.shell(commands=commands)
 621
 622         commands = prepare_venv(version)
 623         stage += hpccm.primitives.shell(commands=commands)
 624
 625         # TODO: Update user home directory.
 626         pyenv_stage += hpccm.primitives.copy(_from=stage_name, _mkdir=True, src=['/root/'],
 627                                              dest='/root')
 628
 629         # Add the intermediate build stage to the sequence
 630         output_stages[stage_name] = stage
 631
 632     # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
 633     # # `version` should be a system installation or pyenv environment (or pyenv-virtualenv)
 634     # # with the dependencies for all of the Python aspects of CMake-driven builds.
 635     # commands = '{pyenv} global {version}'.format(
 636     #             pyenv=pyenv,
 637     #             version=...)
 638     # pyenv_stage += hpccm.primitives.shell(commands=commands)
 639
 640     # Add the aggregating build stage to the sequence. This allows the main stage to copy
 641     # the files in a single stage, potentially reducing the overall output image size.
 642     output_stages['pyenv'] = pyenv_stage
 643
 644
 645 def add_documentation_dependencies(input_args,
 646                                    output_stages: typing.MutableMapping[str, hpccm.Stage]):
 647     """Add appropriate layers according to doxygen input arguments."""
 648     if input_args.doxygen is None:
 649         return
 650     # Always clone the same version of linkchecker (latest release at June 1, 2021)
 651     output_stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=[
 652         'git+https://github.com/linkchecker/linkchecker.git@v10.0.1'])
 653     output_stages['main'] += hpccm.primitives.shell(
 654         commands=[
 655             'sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml'])
 656     if input_args.doxygen == '1.8.5':
 657         doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a'
 658         output_stages['main'] += hpccm.building_blocks.generic_autotools(
 659             repository='https://github.com/westes/flex.git',
 660             commit='f7788a9a0ecccdc953ed12043ccb59ca25714018',
 661             prefix='/tmp/install-of-flex',
 662             configure_opts=['--disable-shared'],
 663             preconfigure=['./autogen.sh'])
 664         output_stages['main'] += hpccm.building_blocks.generic_autotools(
 665             repository='https://github.com/doxygen/doxygen.git',
 666             commit=doxygen_commit,
 667             prefix='',
 668             configure_opts=[
 669                 '--flex /tmp/install-of-flex/bin/flex',
 670                 '--static'])
 671     else:
 672         version = input_args.doxygen
 673         archive_name = f'doxygen-{version}.linux.bin.tar.gz'
 674         archive_url = f'https://sourceforge.net/projects/doxygen/files/rel-{version}/{archive_name}'
 675         binary_path = f'doxygen-{version}/bin/doxygen'
 676         commands = [
 677             'mkdir doxygen && cd doxygen',
 678             f'wget {archive_url}',
 679             f'tar xf {archive_name} {binary_path}',
 680             f'cp {binary_path} /usr/local/bin/',
 681             'cd .. && rm -rf doxygen'
 682         ]
 683         output_stages['main'] += hpccm.primitives.shell(commands=commands)
 684
 685
 686 def build_stages(args) -> typing.Iterable[hpccm.Stage]:
 687     """Define and sequence the stages for the recipe corresponding to *args*."""
 688
 689     # A Dockerfile or Singularity recipe can have multiple build stages.
 690     # The main build stage can copy files from previous stages, though only
 691     # the last stage is included in the tagged output image. This means that
 692     # large or expensive sets of build instructions can be isolated in
 693     # local/temporary images, but all of the stages need to be output by this
 694     # script, and need to occur in the correct order, so we create a sequence
 695     # object early in this function.
 696     stages = collections.OrderedDict()
 697
 698     # If we need TSAN or oneAPI support the early build is more complex,
 699     # so that our compiler images don't have all the cruft needed to get those things
 700     # installed.
 701     if args.llvm is not None and args.tsan is not None:
 702         add_tsan_compiler_build_stage(input_args=args, output_stages=stages)
 703     if args.oneapi is not None:
 704         add_oneapi_compiler_build_stage(input_args=args, output_stages=stages)
 705
 706     # Building blocks are chunks of container-builder instructions that can be
 707     # copied to any build stage with the addition operator.
 708     building_blocks = collections.OrderedDict()
 709     building_blocks['base_packages'] = hpccm.building_blocks.packages(
 710         ospackages=_common_packages)
 711
 712     # These are the most expensive and most reusable layers, so we put them first.
 713     building_blocks['compiler'] = get_compiler(args, compiler_build_stage=stages.get('compiler_build'))
 714     building_blocks['gdrcopy'] = get_gdrcopy(args, building_blocks['compiler'])
 715     building_blocks['ucx'] = get_ucx(args, building_blocks['compiler'], building_blocks['gdrcopy'])
 716     building_blocks['mpi'] = get_mpi(args, building_blocks['compiler'], building_blocks['ucx'])
 717     for i, cmake in enumerate(args.cmake):
 718         building_blocks['cmake' + str(i)] = hpccm.building_blocks.cmake(
 719             eula=True,
 720             prefix=f'/usr/local/cmake-{cmake}',
 721             version=cmake)
 722
 723     # Install additional packages early in the build to optimize Docker build layer cache.
 724     os_packages = list(get_llvm_packages(args)) + get_opencl_packages(args) + get_rocm_packages(args)
 725     if args.doxygen is not None:
 726         os_packages += _docs_extra_packages
 727     if args.oneapi is not None:
 728         os_packages += ['lsb-release']
 729     if args.hipsycl is not None:
 730         os_packages += ['libboost-fiber-dev']
 731     if args.intel_compute_runtime is not None:
 732         os_packages += _intel_compute_runtime_extra_packages
 733     building_blocks['extra_packages'] = []
 734     if args.rocm is not None:
 735         building_blocks['extra_packages'] += hpccm.building_blocks.packages(
 736             apt_keys=['http://repo.radeon.com/rocm/rocm.gpg.key'],
 737             apt_repositories=[f'deb [arch=amd64] http://repo.radeon.com/rocm/apt/{args.rocm}/ xenial main']
 738         )
 739     building_blocks['extra_packages'] += hpccm.building_blocks.packages(
 740         ospackages=os_packages,
 741         apt_ppas=['ppa:intel-opencl/intel-opencl'])
 742
 743     if args.cuda is not None and args.llvm is not None:
 744         # Hack to tell clang what version of CUDA we're using
 745         # based on https://github.com/llvm/llvm-project/blob/1fdec59bffc11ae37eb51a1b9869f0696bfd5312/clang/lib/Driver/ToolChains/Cuda.cpp#L43
 746         cuda_version_split = args.cuda.split('.')
 747         # LLVM requires having the version in x.y.z format, while args.cuda be be either x.y or x.y.z
 748         cuda_version_str = '{}.{}.{}'.format(
 749             cuda_version_split[0],
 750             cuda_version_split[1],
 751             cuda_version_split[2] if len(cuda_version_split) > 2 else 0
 752         )
 753         building_blocks['cuda-clang-workaround'] = hpccm.primitives.shell(commands=[
 754             f'echo "CUDA Version {cuda_version_str}" > /usr/local/cuda/version.txt'
 755         ])
 756
 757     building_blocks['clfft'] = get_clfft(args)
 758
 759     building_blocks['heffte'] = get_heffte(args)
 760
 761     building_blocks['hipSYCL'] = get_hipsycl(args)
 762
 763     building_blocks['intel-compute-runtime'] = get_intel_compute_runtime(args)
 764
 765     # Add Python environments to MPI images, only, so we don't have to worry
 766     # about whether to install mpi4py.
 767     if args.mpi is not None and len(args.venvs) > 0:
 768         add_python_stages(building_blocks=building_blocks, input_args=args, output_stages=stages)
 769
 770     # Create the stage from which the targeted image will be tagged.
 771     stages['main'] = hpccm.Stage()
 772
 773     stages['main'] += hpccm.primitives.baseimage(image=base_image_tag(args))
 774     for bb in building_blocks.values():
 775         if bb is not None:
 776             stages['main'] += bb
 777
 778     # We always add Python3 and Pip
 779     stages['main'] += hpccm.building_blocks.python(python3=True, python2=False)
 780
 781     # Add documentation requirements (doxygen and sphinx + misc).
 782     if args.doxygen is not None:
 783         add_documentation_dependencies(args, stages)
 784
 785     if 'pyenv' in stages and stages['pyenv'] is not None:
 786         stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/.pyenv/'],
 787                                                 dest='/root/.pyenv')
 788         stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/venv/'],
 789                                                 dest='/root/venv')
 790         # TODO: Update user home directory.
 791         # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
 792         # stages['main'] += hpccm.primitives.copy(_from='pyenv', src=['/root/.bashrc'],
 793         #                                         dest='/root/')
 794
 795     # Make sure that `python` resolves to something.
 796     stages['main'] += hpccm.primitives.shell(commands=['test -x /usr/bin/python || '
 797                                                        'update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && '
 798                                                        '/usr/bin/python --version'])
 799
 800     # Note that the list of stages should be sorted in dependency order.
 801     for build_stage in stages.values():
 802         if build_stage is not None:
 803             yield build_stage
 804
 805
 806 if __name__ == '__main__':
 807     args = parser.parse_args()
 808
 809     # Set container specification output format
 810     hpccm.config.set_container_format(args.format)
 811
 812     container_recipe = build_stages(args)
 813
 814     # Output container specification
 815     for stage in container_recipe:
 816         print(stage)