From: Andrey Alekseenko Date: Mon, 31 May 2021 06:26:32 +0000 (+0000) Subject: Add nightly CI test on OneAPI + Intel GPU X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=86835512c48db16ea4359402ec6c7555c1f6f8bb;p=alexxy%2Fgromacs.git Add nightly CI test on OneAPI + Intel GPU --- diff --git a/admin/containers/buildall.sh b/admin/containers/buildall.sh index fe02f4eea3..a5845cf9b9 100755 --- a/admin/containers/buildall.sh +++ b/admin/containers/buildall.sh @@ -20,6 +20,7 @@ args[${#args[@]}]="--llvm 8 --cuda 10.1 --clfft --mpi openmpi" args[${#args[@]}]="--llvm 8 --cuda 11.0 --clfft --mpi openmpi" args[${#args[@]}]="--llvm 9 --clfft --mpi openmpi --ubuntu 18.04" args[${#args[@]}]="--oneapi 2021.1.1" +args[${#args[@]}]="--oneapi 2021.2.0 --intel-compute-runtime 21.21.19914 --ubuntu 20.04" args[${#args[@]}]="--llvm --doxygen --mpi openmpi --venvs 3.7.7" args[${#args[@]}]="--llvm 11 --cuda 11.2.1 --hipsycl 2bc21b677a --ubuntu 20.04" diff --git a/admin/containers/scripted_gmx_docker_builds.py b/admin/containers/scripted_gmx_docker_builds.py index 7425cb29b7..1a5c94ca7b 100755 --- a/admin/containers/scripted_gmx_docker_builds.py +++ b/admin/containers/scripted_gmx_docker_builds.py @@ -116,6 +116,20 @@ _opencl_extra_packages = [ 'clinfo' ] +# Extra packages needed to build Intel Compute Runtime +_intel_compute_runtime_extra_packages = ['pkg-config', + 'libxml2', + 'libxml2-dev', + 'libigc', + 'libigc-dev', + 'libigdgmm11', + 'libigdgmm-dev', + 'libze-loader', + 'libze-loader-dev', + 'ocl-icd-libopencl1', + 'ocl-icd-opencl-dev', + 'opencl-headers'] + # Extra packages needed to build Python installations from source. _python_extra_packages = ['build-essential', 'ca-certificates', @@ -314,6 +328,24 @@ def get_hipsycl(args): cmake_opts=['-DCMAKE_BUILD_TYPE=Release', *cmake_opts], postinstall=postinstall) +def get_intel_compute_runtime(args): + # The only reason we need to build Compute Runtime ourselves is because Intel packages have no DG1 support + # Otherwise, we could have just installed DEB packages from GitHub or Intel PPA + if args.intel_compute_runtime is None: + return None + + cmake_opts = ['-DCMAKE_BUILD_TYPE=Release', + '-DSKIP_UNIT_TESTS=TRUE', + '-DSUPPORT_GEN8=0', '-DSUPPORT_GEN9=1', '-DSUPPORT_GEN11=1', '-DSUPPORT_GEN12LP=1', '-DSUPPORT_DG1=1', + '-DBUILD_WITH_L0=1'] + + return hpccm.building_blocks.generic_cmake( + repository='https://github.com/intel/compute-runtime.git', + directory='compute-runtime', + prefix='/usr/local', recursive=True, branch=args.intel_compute_runtime, + cmake_opts=cmake_opts, + postinstall=['ldconfig']) + def add_tsan_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]): """Isolate the expensive TSAN preparation stage. @@ -575,6 +607,8 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]: os_packages += ['lsb-release'] if args.hipsycl is not None: os_packages += ['libboost-fiber-dev'] + if args.intel_compute_runtime is not None: + os_packages += _intel_compute_runtime_extra_packages building_blocks['extra_packages'] = hpccm.building_blocks.packages( ospackages=os_packages, apt_ppas=['ppa:intel-opencl/intel-opencl'], @@ -600,6 +634,8 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]: building_blocks['hipSYCL'] = get_hipsycl(args) + building_blocks['intel-compute-runtime'] = get_intel_compute_runtime(args) + # Add Python environments to MPI images, only, so we don't have to worry # about whether to install mpi4py. if args.mpi is not None and len(args.venvs) > 0: diff --git a/admin/containers/utility.py b/admin/containers/utility.py index c374f88480..cc123d9195 100644 --- a/admin/containers/utility.py +++ b/admin/containers/utility.py @@ -123,6 +123,9 @@ parser.add_argument('--tsan', type=str, nargs='?', const='llvm', default=None, parser.add_argument('--hipsycl', type=str, nargs='?', default=None, help='Select hipSYCL repository tag/commit/branch.') +parser.add_argument('--intel-compute-runtime', type=str, nargs='?', default=None, + help='Select Intel Compute Runtime version.') + parser.add_argument('--clfft', type=str, nargs='?', const='master', default=None, help='Add external clFFT libraries to the build image') @@ -166,6 +169,8 @@ def image_name(configuration: argparse.Namespace) -> str: elements.append(gpusdk + '-' + version) if configuration.oneapi is not None: elements.append('oneapi-' + configuration.oneapi) + if configuration.intel_compute_runtime is not None: + elements.append('intel-' + configuration.intel_compute_runtime) # Check for special cases # The following attribute keys indicate the image is built for the named diff --git a/admin/gitlab-ci/gromacs.matrix.gitlab-ci.yml b/admin/gitlab-ci/gromacs.matrix.gitlab-ci.yml index f67cfc5a6e..f498dd1a50 100644 --- a/admin/gitlab-ci/gromacs.matrix.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.matrix.gitlab-ci.yml @@ -203,6 +203,9 @@ - if grep -qF 'nvidia.com/gpu' <<< "$KUBERNETES_EXTENDED_RESOURCE_NAME"; then nvidia-smi || true; fi + - if grep -qF 'intel.com/gpu' <<< "$KUBERNETES_EXTENDED_RESOURCE_NAME"; then + sycl-ls || true; + fi - ctest -D $CTEST_RUN_MODE --output-on-failure | tee ctestLog.log || true - awk '/The following tests FAILED/,/^Errors while running CTest|^$/' ctestLog.log | tee ctestErrors.log @@ -275,4 +278,5 @@ include: - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-opencl.gitlab-ci.yml' - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-opencl-release.gitlab-ci.yml' - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-sycl.gitlab-ci.yml' + - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml' diff --git a/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml b/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml new file mode 100644 index 0000000000..7a7ea761df --- /dev/null +++ b/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml @@ -0,0 +1,129 @@ +# Test goal: Newest ICPX build with SYCL with tests on Intel GPUs +# Test intents (should change rarely and conservatively): +# OS: Ubuntu newest supported +# Compiler: ICPX newest supported +# FFT: MKL +# GPU: SYCL +# Scope: configure, build, unit tests, regression tests +# Test implementation choices (free to change as needed): +# OS: Ubuntu 20.04 +# Build type: RelWithAssert +# Compiler: ICPX 2021.2 +# MPI: thread_MPI +# SIMD: AVX2_256 +# Parallelism nt/ntomp: 2/1 + +gromacs:oneapi-2021.2.0-sycl-nightly:configure: + extends: + - .gromacs:base:configure + - .use-oneapi:base + - .use-sycl + - .rules:nightly-not-for-release + image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest + variables: + CMAKE: /usr/local/cmake-3.18.4/bin/cmake + COMPILER_MAJOR_VERSION: 2021 + +gromacs:oneapi-2021.2.0-sycl-nightly:build: + extends: + - .variables:default + - .gromacs:base:build + - .use-ccache + - .use-oneapi:base + - .rules:nightly-not-for-release + image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest + variables: + CMAKE: /usr/local/cmake-3.18.4/bin/cmake + needs: + - job: gromacs:oneapi-2021.2.0-sycl-nightly:configure + +gromacs:oneapi-2021.2.0-sycl-nightly:test-opencl: + extends: + - .gromacs:base:test + - .use-oneapi:base + - .rules:nightly-not-for-release + tags: + - intel-gpu-dg1 + image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest + variables: + CMAKE: /usr/local/cmake-3.18.4/bin/cmake + SYCL_DEVICE_FILTER: "opencl:gpu" + # The following two variables are not used by k8s, since this job runs on a separate host + # But it's easier to set them up and use the existing scripts for job preparation that rely on them + KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2 + # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged + GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2 + needs: + - job: gromacs:oneapi-2021.2.0-sycl-nightly:build + +gromacs:oneapi-2021.2.0-sycl-nightly:regressiontest-opencl: + extends: + - .gromacs:base:regressiontest + - .use-oneapi:base + - .rules:nightly-not-for-release + tags: + - intel-gpu-dg1 + image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest + variables: + CMAKE: /usr/local/cmake-3.18.4/bin/cmake + SYCL_DEVICE_FILTER: "opencl:gpu" + # Default not working, see #2980 + REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 + REGRESSIONTEST_OMP_RANK_NUMBER: 1 + # The following two variables are not used by k8s, since this job runs on a separate host + # But it's easier to set them up and use the existing scripts for job preparation that rely on them + KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu" + KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2 + # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged + GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2 + needs: + - job: gromacs:oneapi-2021.2.0-sycl-nightly:build + - job: regressiontests:prepare + +# We have to disable LevelZero tests until https://github.com/intel/llvm/pull/3045 makes it into +# the OneAPI release. Without this patch, some of the tests deadlock. + +#gromacs:oneapi-2021.2.0-sycl-nightly:test-levelzero: +# extends: +# - .gromacs:base:test +# - .use-oneapi:base +# - .rules:nightly-not-for-release +# tags: +# - intel-gpu-dg1 +# image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest +# variables: +# CMAKE: /usr/local/cmake-3.18.4/bin/cmake +# SYCL_DEVICE_FILTER: "level_zero:gpu" +# # The following two variables are not used by k8s, since this job runs on a separate host +# # But it's easier to set them up and use the existing scripts for job preparation that rely on them +# KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu" +# KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2 +# # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged +# GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2 +# needs: +# - job: gromacs:oneapi-2021.2.0-sycl-nightly:build +# +#gromacs:oneapi-2021.2.0-sycl-nightly:regressiontest-levelzero: +# extends: +# - .gromacs:base:regressiontest +# - .use-oneapi:base +# - .rules:nightly-not-for-release +# tags: +# - intel-gpu-dg1 +# image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest +# variables: +# CMAKE: /usr/local/cmake-3.18.4/bin/cmake +# SYCL_DEVICE_FILTER: "level_zero:gpu" +# # Default not working, see #2980 +# REGRESSIONTEST_TOTAL_RANK_NUMBER: 2 +# REGRESSIONTEST_OMP_RANK_NUMBER: 1 +# # The following two variables are not used by k8s, since this job runs on a separate host +# # But it's easier to set them up and use the existing scripts for job preparation that rely on them +# KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu" +# KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2 +# # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged +# GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2 +# needs: +# - job: gromacs:oneapi-2021.2.0-sycl-nightly:build +# - job: regressiontests:prepare