Prepare changes to docker images for python packages

Python package handling for things currently added manually to the
system python are moved to the virtual environments instead. This
affects the sphinx and gcovr packages.

diff --git a/admin/containers/buildall.sh b/admin/containers/buildall.sh
index 95403fda6d28e9c43c1c05d07d6fbf0484a87206..71adef68d3355c83f1cfbb27208cdf0de9eb1cc6 100644 (file)
--- a/admin/containers/buildall.sh
+++ b/admin/containers/buildall.sh
@@ -20,7 +20,7 @@ args[${#args[@]}]="--llvm 8 --cuda 10.1 --clfft --mpi openmpi"
 args[${#args[@]}]="--llvm 8 --cuda 11.0 --clfft --mpi openmpi"
 args[${#args[@]}]="--llvm 9 --clfft --mpi openmpi --ubuntu 18.04"
 args[${#args[@]}]="--oneapi 2021.1.1"
-args[${#args[@]}]="--llvm --doxygen"
+args[${#args[@]}]="--llvm --doxygen --mpi openmpi --venvs 3.7.7"
 
 echo "Building the following images."
 for arg_string in "${args[@]}"; do

diff --git a/admin/containers/scripted_gmx_docker_builds.py b/admin/containers/scripted_gmx_docker_builds.py
index c348094f2dc8efa709900743ad7e7c4d03c05487..4dc5855f6646915da2955ded0c2d26403a54d275 100755 (executable)
--- a/admin/containers/scripted_gmx_docker_builds.py
+++ b/admin/containers/scripted_gmx_docker_builds.py
@@ -223,8 +223,8 @@ def get_compiler(args, compiler_build_stage: hpccm.Stage = None) -> bb_base:
             compiler = compiler_build_stage.runtime(_from='oneapi')
             # Prepare the toolchain (needed only for builds done within the Dockerfile, e.g.
             # OpenMPI builds, which don't currently work for other reasons)
-            oneapi_toolchain = hpccm.toolchain(CC='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icx',
-                                               CXX='/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icpx')
+            oneapi_toolchain = hpccm.toolchain(CC=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icx',
+                                               CXX=f'/opt/intel/oneapi/compiler/{args.oneapi}/linux/bin/intel64/icpx')
             setattr(compiler, 'toolchain', oneapi_toolchain)
 
         else:
@@ -344,7 +344,9 @@ def add_oneapi_compiler_build_stage(input_args, output_stages: typing.Mapping[st
     )
     # Ensure that all bash shells on the final container will have access to oneAPI
     oneapi_stage += hpccm.primitives.shell(
-            commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc']
+            commands=['echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc',
+                      'unlink /opt/intel/oneapi/compiler/latest',
+                     f'ln -sf /opt/intel/oneapi/compiler/{version} /opt/intel/oneapi/compiler/latest']
             )
     setattr(oneapi_stage, 'runtime', oneapi_runtime)
 
@@ -368,13 +370,17 @@ def prepare_venv(version: StrictVersion) -> typing.Sequence[str]:
     commands.append(f"""{venv_path}/bin/python -m pip install --upgrade \
             'cmake>=3.13' \
             'flake8>=3.7.7' \
+            'gcovr>=4.2' \
             'mpi4py>=3.0.3' \
             'networkx>=2.0' \
             'numpy>=1' \
             'pip>=10.1' \
+            'Pygments>=2.2.0' \
             'pytest>=3.9' \
             'setuptools>=42' \
-            'scikit-build>=0.10'""")
+            'scikit-build>=0.10' \
+            'Sphinx>=1.6.3' \
+            'sphinxcontrib-plantuml>=0.14'""")
 
     # TODO: Remove 'importlib_resources' dependency when Python >=3.7 is required.
     if minor == 6:
@@ -459,7 +465,6 @@ def add_documentation_dependencies(input_args,
         return
     output_stages['main'] += hpccm.primitives.shell(
         commands=['sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml'])
-    output_stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1', 'gcovr'])
     if input_args.doxygen == '1.8.5':
         doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a'
         output_stages['main'] += hpccm.building_blocks.generic_autotools(
@@ -554,9 +559,7 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]:
             stages['main'] += bb
 
     # We always add Python3 and Pip
-    stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True)
-    stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3',
-                                                packages=['pytest', 'networkx', 'numpy'])
+    stages['main'] += hpccm.building_blocks.python(python3=True, python2=False)
 
     # Add documentation requirements (doxygen and sphinx + misc).
     if args.doxygen is not None:

diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml
index 36ce9b6ea3cd337bfdc36eafde71a3baa965a8e6..25a53dbff9013a6b42470e3fb307b17ddaf6e5c0 100644 (file)
--- a/admin/gitlab-ci/documentation.gitlab-ci.yml
+++ b/admin/gitlab-ci/documentation.gitlab-ci.yml
@@ -42,6 +42,11 @@
       mkdir $BUILD_DIR ;
       fi
     - cd $BUILD_DIR
+    # Running CMake with the venv activated should not be strictly necessary,
+    # but helps to find and cache self-consistent Python and Sphinx details
+    # without additional hinting. Once CMakeCache.txt exists, the later stages
+    # should work fine without reactivating the venv.
+    - source /root/venv/py3.7/bin/activate
     - cmake ..
       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
       $CMAKE_COMPILER_SCRIPT

diff --git a/docs/dev-manual/gitlab.rst b/docs/dev-manual/gitlab.rst
index 3c5dda31de7c3e62440304fa019da8f8c61eac44..6f12994ed669e556a70bf8306d67912363adfffa 100644 (file)
--- a/docs/dev-manual/gitlab.rst
+++ b/docs/dev-manual/gitlab.rst
@@ -59,7 +59,7 @@ GitLab CI job parameters, but note the following GROMACS-specific conventions.
         Avoid using *before-script* directly, and be cautious
         about nested *extends* overriding multiple *before_script* definitions.
 
-    cache
+    job cache
         There is no global default, but jobs that build software will likely
         set *cache*. To explicitly unset *cache* directives, specify a job
         parameter of ``cache: {}``.