From: Paul Bauer Date: Thu, 11 Feb 2021 09:21:32 +0000 (+0000) Subject: Prepare changes to docker images for python packages X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=c0981ea1fed5873407ef6404e3d6097a6b1b0a99;p=alexxy%2Fgromacs.git Prepare changes to docker images for python packages Python package handling for things currently added manually to the system python are moved to the virtual environments instead. This affects the sphinx and gcovr packages. --- diff --git a/admin/containers/buildall.sh b/admin/containers/buildall.sh index 95403fda6d..71adef68d3 100644 --- a/admin/containers/buildall.sh +++ b/admin/containers/buildall.sh @@ -20,7 +20,7 @@ args[${#args[@]}]="--llvm 8 --cuda 10.1 --clfft --mpi openmpi" args[${#args[@]}]="--llvm 8 --cuda 11.0 --clfft --mpi openmpi" args[${#args[@]}]="--llvm 9 --clfft --mpi openmpi --ubuntu 18.04" args[${#args[@]}]="--oneapi 2021.1.1" -args[${#args[@]}]="--llvm --doxygen" +args[${#args[@]}]="--llvm --doxygen --mpi openmpi --venvs 3.7.7" echo "Building the following images." for arg_string in "${args[@]}"; do diff --git a/admin/containers/scripted_gmx_docker_builds.py b/admin/containers/scripted_gmx_docker_builds.py index c348094f2d..4dc5855f66 100755 --- a/admin/containers/scripted_gmx_docker_builds.py +++ b/admin/containers/scripted_gmx_docker_builds.py @@ -223,8 +223,8 @@ def get_compiler(args, compiler_build_stage: hpccm.Stage = None) -> bb_base: compiler = compiler_build_stage.runtime(_from='oneapi') # Prepare the toolchain (needed only for builds done within the Dockerfile, e.g. # OpenMPI builds, which don't currently work for other reasons) - oneapi_toolchain = hpccm.toolchain(CC='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icx', - CXX='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icpx') + oneapi_toolchain = hpccm.toolchain(CC=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icx', + CXX=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icpx') setattr(compiler, 'toolchain', oneapi_toolchain) else: @@ -344,7 +344,9 @@ def add_oneapi_compiler_build_stage(input_args, output_stages: typing.Mapping[st ) # Ensure that all bash shells on the final container will have access to oneAPI oneapi_stage += hpccm.primitives.shell( - commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc'] + commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc', + 'unlink /opt/intel/oneapi/compiler/latest', + f'ln -sf /opt/intel/oneapi/compiler/{version} /opt/intel/oneapi/compiler/latest'] ) setattr(oneapi_stage, 'runtime', oneapi_runtime) @@ -368,13 +370,17 @@ def prepare_venv(version: StrictVersion) -> typing.Sequence[str]: commands.append(f"""{venv_path}/bin/python -m pip install --upgrade \ 'cmake>=3.13' \ 'flake8>=3.7.7' \ + 'gcovr>=4.2' \ 'mpi4py>=3.0.3' \ 'networkx>=2.0' \ 'numpy>=1' \ 'pip>=10.1' \ + 'Pygments>=2.2.0' \ 'pytest>=3.9' \ 'setuptools>=42' \ - 'scikit-build>=0.10'""") + 'scikit-build>=0.10' \ + 'Sphinx>=1.6.3' \ + 'sphinxcontrib-plantuml>=0.14'""") # TODO: Remove 'importlib_resources' dependency when Python >=3.7 is required. if minor == 6: @@ -459,7 +465,6 @@ def add_documentation_dependencies(input_args, return output_stages['main'] += hpccm.primitives.shell( commands=['sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml']) - output_stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1', 'gcovr']) if input_args.doxygen == '1.8.5': doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a' output_stages['main'] += hpccm.building_blocks.generic_autotools( @@ -554,9 +559,7 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]: stages['main'] += bb # We always add Python3 and Pip - stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True) - stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3', - packages=['pytest', 'networkx', 'numpy']) + stages['main'] += hpccm.building_blocks.python(python3=True, python2=False) # Add documentation requirements (doxygen and sphinx + misc). if args.doxygen is not None: diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 36ce9b6ea3..25a53dbff9 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -42,6 +42,11 @@ mkdir $BUILD_DIR ; fi - cd $BUILD_DIR + # Running CMake with the venv activated should not be strictly necessary, + # but helps to find and cache self-consistent Python and Sphinx details + # without additional hinting. Once CMakeCache.txt exists, the later stages + # should work fine without reactivating the venv. + - source /root/venv/py3.7/bin/activate - cmake .. -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache $CMAKE_COMPILER_SCRIPT diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst index 3c5dda31de..6f12994ed6 100644 --- a/docs/dev-manual/gitlab.rst +++ b/docs/dev-manual/gitlab.rst @@ -59,7 +59,7 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions. Avoid using *before-script* directly, and be cautious about nested *extends* overriding multiple *before_script* definitions. - cache + job cache There is no global default, but jobs that build software will likely set *cache*. To explicitly unset *cache* directives, specify a job parameter of ``cache: {}``.