3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2020, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
37 Generates a set of docker images used for running GROMACS CI on Gitlab.
38 The images are prepared according to a selection of build configuration targets
39 that hope to cover a broad enough scope of different possible systems,
40 allowing us to check compiler types and versions, as well as libraries used
41 for accelerators and parallel communication systems. Each combinations is
42 described as an entry in the build_configs dictionary, with the script
43 analysing the logic and adding build stages as needed.
45 Based on the example script provided by the NVidia HPCCM repository.
48 * Paul Bauer <paul.bauer.q@gmail.com>
49 * Eric Irrgang <ericirrgang@gmail.com>
50 * Joe Jordan <e.jjordan12@gmail.com>
54 $ python3 scripted_gmx_docker_builds.py --help
55 $ python3 scripted_gmx_docker_builds.py --format docker > Dockerfile && docker build .
56 $ python3 scripted_gmx_docker_builds.py | docker build -
63 from distutils.version import StrictVersion
67 from hpccm.building_blocks.base import bb_base
73 'This module assumes availability of supporting modules in the same directory. Add the directory to '
74 'PYTHONPATH or invoke Python from within the module directory so module location can be resolved.')
76 # Basic packages for all final images.
77 _common_packages = ['build-essential',
93 # Extra packages needed to build Python installations from source.
94 _python_extra_packages = ['build-essential',
113 # Extra packages needed for images for building documentation.
114 _docs_extra_packages = ['autoconf',
129 'texlive-latex-base',
130 'texlive-latex-extra',
131 'texlive-fonts-recommended',
132 'texlive-fonts-extra']
134 # Supported Python versions for maintained branches.
135 # TODO: Remove '3.5.9' from defaults in master once script in release-2020 diverges.
136 _python_versions = ['3.5.9', '3.6.10', '3.7.7', '3.8.2']
138 # Parse command line arguments
139 parser = argparse.ArgumentParser(description='GROMACS CI image creation script', parents=[utility.parser])
141 parser.add_argument('--format', type=str, default='docker',
142 choices=['docker', 'singularity'],
143 help='Container specification format (default: docker)')
144 parser.add_argument('--venvs', nargs='*', type=str, default=_python_versions,
145 help='List of Python versions ("major.minor.patch") for which to install venvs. '
146 'Default: {}'.format(' '.join(_python_versions)))
149 def base_image_tag(args) -> str:
150 # Check if we use CUDA images or plain linux images
151 if args.cuda is not None:
152 cuda_version_tag = 'nvidia/cuda:' + args.cuda + '-devel'
153 if args.centos is not None:
154 cuda_version_tag += '-centos' + args.centos
155 elif args.ubuntu is not None:
156 cuda_version_tag += '-ubuntu' + args.ubuntu
158 raise RuntimeError('Logic error: no Linux distribution selected.')
160 base_image_tag = cuda_version_tag
162 if args.centos is not None:
163 base_image_tag = 'centos:centos' + args.centos
164 elif args.ubuntu is not None:
165 base_image_tag = 'ubuntu:' + args.ubuntu
167 raise RuntimeError('Logic error: no Linux distribution selected.')
168 return base_image_tag
171 def get_llvm_packages(args) -> typing.Iterable[str]:
172 # If we use the package version of LLVM, we need to install extra packages for it.
173 if (args.llvm is not None) and (args.tsan is None):
174 return ['libomp-dev',
175 'clang-format-' + str(args.llvm),
176 'clang-tidy-' + str(args.llvm)]
181 def get_compiler(args, tsan_stage: hpccm.Stage = None) -> bb_base:
183 if args.icc is not None:
184 raise RuntimeError('Intel compiler toolchain recipe not implemented yet')
186 if args.llvm is not None:
187 # Build our own version instead to get TSAN + OMP
188 if args.tsan is not None:
189 if tsan_stage is not None:
190 compiler = tsan_stage.runtime(_from='tsan')
192 raise RuntimeError('No TSAN stage!')
193 # Build the default compiler if we don't need special support
195 compiler = hpccm.building_blocks.llvm(extra_repository=True, version=args.llvm)
197 elif (args.gcc is not None):
198 compiler = hpccm.building_blocks.gnu(extra_repository=True,
202 raise RuntimeError('Logic error: no compiler toolchain selected.')
206 def get_mpi(args, compiler):
207 # If needed, add MPI to the image
208 if args.mpi is not None:
209 if args.mpi == 'openmpi':
211 if args.cuda is not None:
214 if hasattr(compiler, 'toolchain'):
215 return hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, cuda=use_cuda, infiniband=False)
217 raise RuntimeError('compiler is not an HPCCM compiler building block!')
219 elif args.mpi == 'impi':
220 raise RuntimeError('Intel MPI recipe not implemented yet.')
222 raise RuntimeError('Requested unknown MPI implementation.')
227 def get_opencl(args):
228 # Add OpenCL environment if needed
229 if (args.opencl is not None):
230 if args.opencl == 'nvidia':
231 if (args.cuda is None):
232 raise RuntimeError('Need Nvidia environment for Nvidia OpenCL image')
234 return hpccm.building_blocks.packages(ospackages=['nvidia-opencl-dev'])
236 elif args.opencl == 'intel':
237 return hpccm.building_blocks.packages(ospackages=['ocl-icd-opencl-dev', 'opencl-headers',
238 'beignet-opencl-icd'])
239 elif args.opencl == 'amd':
240 # Due to the wisdom of AMD, this needs to be done differently for the OS and version! Hurray!
241 # And they don't allow wget, so this branch is not taken for now! AMD, please allow me to use wget.
243 'AMD recipe can not be generated because they do not allow wget for getting the packages.')
245 # if args.ubuntu is not '16.04':
246 # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/ubuntu/'+args.ubuntu+'/amdgpu-pro-18.30-641594.tar.xz',
247 # install=['./amdgpu-install --opencl=legacy --headless -y'])
249 # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/ubuntu/amdgpu-pro-18.30-641594.tar.xz',
250 # install=['./amdgpu-install --opencl=legacy --headless -y'])
252 # Stage0 += hpccm.building_blocks.generic_build(url='https://www2.ati.com/drivers/linux/rhel'+args.centos'/amdgpu-pro-18.30-641594.tar.xz',
253 # install=['./amdgpu-install --opencl=legacy --headless -y'])
259 if (args.clfft is not None):
260 return hpccm.building_blocks.generic_cmake(
261 repository='https://github.com/clMathLibraries/clFFT.git',
262 prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src')
267 def add_tsan_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
268 """Isolate the expensive TSAN preparation stage.
270 This is a very expensive stage, but has few and disjoint dependencies, and
271 its output is easily compartmentalized (/usr/local) so we can isolate this
272 build stage to maximize build cache hits and reduce rebuild time, bookkeeping,
273 and final image size.
275 if not isinstance(output_stages, collections.abc.MutableMapping):
276 raise RuntimeError('Need output_stages container.')
277 tsan_stage = hpccm.Stage()
278 tsan_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='tsan')
280 tsan_stage += hpccm.building_blocks.packages(ospackages=['git', 'ca-certificates', 'build-essential', 'cmake'])
281 # CMake will get duplicated later, but this is an expensive image, and it isn't worth optimizing
282 # out that duplication...
283 tsan_stage += hpccm.building_blocks.python(python3=True, python2=False, devel=False)
285 compiler_branch = 'release_' + str(input_args.llvm) + '0'
286 tsan_stage += hpccm.building_blocks.generic_cmake(
287 repository='https://git.llvm.org/git/llvm.git',
288 prefix='/usr/local', recursive=True, branch=compiler_branch,
289 cmake_opts=['-D CMAKE_BUILD_TYPE=Release', '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra"',
290 '-D LIBOMP_TSAN_SUPPORT=on'],
291 preconfigure=['export branch=' + compiler_branch,
292 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxx.git)',
293 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxxabi.git)',
294 '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/compiler-rt.git)',
295 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/openmp.git)',
296 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang.git)',
297 '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang-tools-extra.git)'],
298 postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-' + str(input_args.llvm),
299 'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-' + str(input_args.llvm),
300 'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-' + str(input_args.llvm),
301 'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-' + str(input_args.llvm)])
302 output_stages['tsan'] = tsan_stage
305 def prepare_venv(version: StrictVersion) -> typing.Sequence[str]:
306 """Get shell commands to set up the venv for the requested Python version."""
307 major = version.version[0]
308 minor = version.version[1]
310 pyenv = '$HOME/.pyenv/bin/pyenv'
312 py_ver = '{}.{}'.format(major, minor)
313 venv_path = '$HOME/venv/py{}'.format(py_ver)
314 commands = ['$({pyenv} prefix `{pyenv} whence python{py_ver}`)/bin/python -m venv {path}'.format(
320 commands.append('{path}/bin/python -m pip install --upgrade pip setuptools'.format(
323 # Install dependencies for building and testing gmxapi Python package.
324 # WARNING: Please keep this list synchronized with python_packaging/requirements-test.txt
325 # TODO: Get requirements.txt from an input argument.
326 commands.append("""{path}/bin/python -m pip install --upgrade \
334 'setuptools>=28.0.0' \
335 'scikit-build>=0.7'""".format(path=venv_path))
340 def add_python_stages(building_blocks: typing.Mapping[str, bb_base],
342 output_stages: typing.MutableMapping[str, hpccm.Stage]):
343 """Add the stage(s) necessary for the requested venvs.
345 One intermediate build stage is created for each venv (see --venv option).
347 Each stage partially populates Python installations and venvs in the home
348 directory. The home directory is collected by the 'pyenv' stage for use by
349 the main build stage.
351 if len(input_args.venvs) < 1:
352 raise RuntimeError('No venvs to build...')
353 if output_stages is None or not isinstance(output_stages, collections.abc.Mapping):
354 raise RuntimeError('Need a container for output stages.')
356 # Main Python stage that collects the environments from individual stages.
357 # We collect the stages individually, rather than chaining them, because the
358 # copy is a bit slow and wastes local Docker image space for each filesystem
360 pyenv_stage = hpccm.Stage()
361 pyenv_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='pyenv')
362 pyenv_stage += building_blocks['compiler']
363 pyenv_stage += building_blocks['mpi']
364 pyenv_stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
366 for version in [StrictVersion(py_ver) for py_ver in sorted(input_args.venvs)]:
367 stage_name = 'py' + str(version)
368 stage = hpccm.Stage()
369 stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as=stage_name)
370 stage += building_blocks['compiler']
371 stage += building_blocks['mpi']
372 stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
374 # TODO: Use a non-root user for testing and Python virtual environments.
375 stage += hpccm.primitives.shell(commands=[
376 'curl https://pyenv.run | bash',
377 """echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $HOME/.bashrc""",
378 """echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> $HOME/.bashrc""",
379 """echo 'eval "$(pyenv init -)"' >> $HOME/.bashrc""",
380 """echo 'eval "$(pyenv virtualenv-init -)"' >> $HOME/.bashrc"""])
381 pyenv = '$HOME/.pyenv/bin/pyenv'
382 commands = ['PYTHON_CONFIGURE_OPTS="--enable-shared" {pyenv} install -s {version}'.format(
384 version=str(version))]
385 stage += hpccm.primitives.shell(commands=commands)
387 commands = prepare_venv(version)
388 stage += hpccm.primitives.shell(commands=commands)
390 # TODO: Update user home directory.
391 pyenv_stage += hpccm.primitives.copy(_from=stage_name, _mkdir=True, src=['/root/'],
394 # Add the intermediate build stage to the sequence
395 output_stages[stage_name] = stage
397 # Add the aggregating build stage to the sequence. This allows the main stage to copy
398 # the files in a single stage, potentially reducing the overall output image size.
399 output_stages['pyenv'] = pyenv_stage
402 def build_stages(args) -> typing.Iterable[hpccm.Stage]:
403 """Define and sequence the stages for the recipe corresponding to *args*."""
405 # A Dockerfile or Singularity recipe can have multiple build stages.
406 # The main build stage can copy files from previous stages, though only
407 # the last stage is included in the tagged output image. This means that
408 # large or expensive sets of build instructions can be isolated in
409 # local/temporary images, but all of the stages need to be output by this
410 # script, and need to occur in the correct order, so we create a sequence
411 # object early in this function.
412 stages = collections.OrderedDict()
414 # If we need the TSAN compilers, the early build is more involved.
415 if args.llvm is not None and args.tsan is not None:
416 add_tsan_stage(input_args=args, output_stages=stages)
418 # Building blocks are chunks of container-builder instructions that can be
419 # copied to any build stage with the addition operator.
420 building_blocks = collections.OrderedDict()
422 # These are the most expensive and most reusable layers, so we put them first.
423 building_blocks['compiler'] = get_compiler(args, tsan_stage=stages.get('tsan'))
424 building_blocks['mpi'] = get_mpi(args, building_blocks['compiler'])
426 # Install additional packages early in the build to optimize Docker build layer cache.
427 os_packages = _common_packages + get_llvm_packages(args)
428 if args.doxygen is not None:
429 os_packages += _docs_extra_packages
430 building_blocks['ospackages'] = hpccm.building_blocks.packages(ospackages=os_packages)
432 building_blocks['cmake'] = hpccm.building_blocks.cmake(eula=True, version=args.cmake)
433 building_blocks['opencl'] = get_opencl(args)
434 building_blocks['clfft'] = get_clfft(args)
436 # Add Python environments to MPI images, only, so we don't have to worry
437 # about whether to install mpi4py.
438 if args.mpi is not None and len(args.venvs) > 0:
439 add_python_stages(building_blocks=building_blocks, input_args=args, output_stages=stages)
441 # Create the stage from which the targeted image will be tagged.
442 stages['main'] = hpccm.Stage()
444 stages['main'] += hpccm.primitives.baseimage(image=base_image_tag(args))
445 for bb in building_blocks.values():
449 # We always add Python3 and Pip
450 stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True)
451 stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3',
452 packages=['pytest', 'networkx', 'numpy'])
454 # Add documentation requirements (doxygen and sphinx + misc).
455 if (args.doxygen is not None):
456 if (args.doxygen == '1.8.5'):
457 doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a'
459 doxygen_commit = 'a6d4f4df45febe588c38de37641513fd576b998f'
460 stages['main'] += hpccm.building_blocks.generic_autotools(
461 repository='https://github.com/westes/flex.git',
462 commit='f7788a9a0ecccdc953ed12043ccb59ca25714018',
463 prefix='/tmp/install-of-flex',
464 configure_opts=['--disable-shared'],
465 preconfigure=['./autogen.sh'])
466 stages['main'] += hpccm.building_blocks.generic_autotools(
467 repository='https://github.com/doxygen/doxygen.git',
468 commit=doxygen_commit,
471 '--flex /tmp/install-of-flex/bin/flex',
474 'sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml'])
475 stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1'])
477 if 'pyenv' in stages and stages['pyenv'] is not None:
478 # TODO: Update user home directory.
479 stages['main'] += hpccm.primitives.copy(_from='pyenv', src=['/root/.bashrc'],
481 stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/.pyenv/'],
483 stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/venv/'],
486 # Note that the list of stages should be sorted in dependency order.
487 for build_stage in stages.values():
488 if build_stage is not None:
492 if __name__ == '__main__':
493 args = parser.parse_args()
495 # Set container specification output format
496 hpccm.config.set_container_format(args.format)
498 container_recipe = build_stages(args)
500 # Output container specification
501 for stage in container_recipe: