Add nightly CI test on OneAPI + Intel GPU
authorAndrey Alekseenko <al42and@gmail.com>
Mon, 31 May 2021 06:26:32 +0000 (06:26 +0000)
committerPaul Bauer <paul.bauer.q@gmail.com>
Mon, 31 May 2021 06:26:32 +0000 (06:26 +0000)
admin/containers/buildall.sh
admin/containers/scripted_gmx_docker_builds.py
admin/containers/utility.py
admin/gitlab-ci/gromacs.matrix.gitlab-ci.yml
admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml [new file with mode: 0644]

index fe02f4eea325c94463f2c4180a6210daad4269cd..a5845cf9b931cfe392d9df8ee7c743dd0cdec267 100755 (executable)
@@ -20,6 +20,7 @@ args[${#args[@]}]="--llvm 8 --cuda 10.1 --clfft --mpi openmpi"
 args[${#args[@]}]="--llvm 8 --cuda 11.0 --clfft --mpi openmpi"
 args[${#args[@]}]="--llvm 9 --clfft --mpi openmpi --ubuntu 18.04"
 args[${#args[@]}]="--oneapi 2021.1.1"
+args[${#args[@]}]="--oneapi 2021.2.0 --intel-compute-runtime 21.21.19914 --ubuntu 20.04"
 args[${#args[@]}]="--llvm --doxygen --mpi openmpi --venvs 3.7.7"
 args[${#args[@]}]="--llvm 11 --cuda 11.2.1 --hipsycl 2bc21b677a --ubuntu 20.04"
 
index 7425cb29b7ff32ca8bdfaaa577f181ad7f204e38..1a5c94ca7b8f3c9c19b2c5fbbcf7664b52cf91d9 100755 (executable)
@@ -116,6 +116,20 @@ _opencl_extra_packages = [
     'clinfo'
 ]
 
+# Extra packages needed to build Intel Compute Runtime
+_intel_compute_runtime_extra_packages = ['pkg-config',
+                                         'libxml2',
+                                         'libxml2-dev',
+                                         'libigc',
+                                         'libigc-dev',
+                                         'libigdgmm11',
+                                         'libigdgmm-dev',
+                                         'libze-loader',
+                                         'libze-loader-dev',
+                                         'ocl-icd-libopencl1',
+                                         'ocl-icd-opencl-dev',
+                                         'opencl-headers']
+
 # Extra packages needed to build Python installations from source.
 _python_extra_packages = ['build-essential',
                           'ca-certificates',
@@ -314,6 +328,24 @@ def get_hipsycl(args):
         cmake_opts=['-DCMAKE_BUILD_TYPE=Release', *cmake_opts],
         postinstall=postinstall)
 
+def get_intel_compute_runtime(args):
+    # The only reason we need to build Compute Runtime ourselves is because Intel packages have no DG1 support
+    # Otherwise, we could have just installed DEB packages from GitHub or Intel PPA
+    if args.intel_compute_runtime is None:
+        return None
+
+    cmake_opts = ['-DCMAKE_BUILD_TYPE=Release',
+                  '-DSKIP_UNIT_TESTS=TRUE',
+                  '-DSUPPORT_GEN8=0', '-DSUPPORT_GEN9=1', '-DSUPPORT_GEN11=1', '-DSUPPORT_GEN12LP=1', '-DSUPPORT_DG1=1',
+                  '-DBUILD_WITH_L0=1']
+
+    return hpccm.building_blocks.generic_cmake(
+        repository='https://github.com/intel/compute-runtime.git',
+        directory='compute-runtime',
+        prefix='/usr/local', recursive=True, branch=args.intel_compute_runtime,
+        cmake_opts=cmake_opts,
+        postinstall=['ldconfig'])
+
 def add_tsan_compiler_build_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
     """Isolate the expensive TSAN preparation stage.
 
@@ -575,6 +607,8 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]:
         os_packages += ['lsb-release']
     if args.hipsycl is not None:
         os_packages += ['libboost-fiber-dev']
+    if args.intel_compute_runtime is not None:
+        os_packages += _intel_compute_runtime_extra_packages
     building_blocks['extra_packages'] = hpccm.building_blocks.packages(
         ospackages=os_packages,
         apt_ppas=['ppa:intel-opencl/intel-opencl'],
@@ -600,6 +634,8 @@ def build_stages(args) -> typing.Iterable[hpccm.Stage]:
 
     building_blocks['hipSYCL'] = get_hipsycl(args)
 
+    building_blocks['intel-compute-runtime'] = get_intel_compute_runtime(args)
+
     # Add Python environments to MPI images, only, so we don't have to worry
     # about whether to install mpi4py.
     if args.mpi is not None and len(args.venvs) > 0:
index c374f88480f065a38483218ec360140542f081d3..cc123d9195b47714fbe39e20eaff38a722def3c9 100644 (file)
@@ -123,6 +123,9 @@ parser.add_argument('--tsan', type=str, nargs='?', const='llvm', default=None,
 parser.add_argument('--hipsycl', type=str, nargs='?', default=None,
                     help='Select hipSYCL repository tag/commit/branch.')
 
+parser.add_argument('--intel-compute-runtime', type=str, nargs='?', default=None,
+                    help='Select Intel Compute Runtime version.')
+
 parser.add_argument('--clfft', type=str, nargs='?', const='master', default=None,
                     help='Add external clFFT libraries to the build image')
 
@@ -166,6 +169,8 @@ def image_name(configuration: argparse.Namespace) -> str:
             elements.append(gpusdk + '-' + version)
     if configuration.oneapi is not None:
         elements.append('oneapi-' + configuration.oneapi)
+    if configuration.intel_compute_runtime is not None:
+        elements.append('intel-' + configuration.intel_compute_runtime)
 
     # Check for special cases
     # The following attribute keys indicate the image is built for the named
index f67cfc5a6e0cfddadf91429f16433a1dbd326512..f498dd1a50cd5b73f70ef9d31ef26cd6fdb606bb 100644 (file)
     - if grep -qF 'nvidia.com/gpu' <<< "$KUBERNETES_EXTENDED_RESOURCE_NAME"; then
       nvidia-smi || true;
       fi
+    - if grep -qF 'intel.com/gpu' <<< "$KUBERNETES_EXTENDED_RESOURCE_NAME"; then
+      sycl-ls || true;
+      fi
     - ctest -D $CTEST_RUN_MODE --output-on-failure | tee ctestLog.log || true
     - awk '/The following tests FAILED/,/^Errors while running CTest|^$/'
       ctestLog.log | tee ctestErrors.log
@@ -275,4 +278,5 @@ include:
   - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-opencl.gitlab-ci.yml'
   - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-opencl-release.gitlab-ci.yml'
   - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.1.1-sycl.gitlab-ci.yml'
+  - local: '/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml'
 
diff --git a/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml b/admin/gitlab-ci/gromacs.matrix/gromacs.oneapi-2021.2.0-sycl-nightly.gitlab-ci.yml
new file mode 100644 (file)
index 0000000..7a7ea76
--- /dev/null
@@ -0,0 +1,129 @@
+# Test goal: Newest ICPX build with SYCL with tests on Intel GPUs
+# Test intents (should change rarely and conservatively):
+#   OS: Ubuntu newest supported
+#   Compiler: ICPX newest supported
+#   FFT: MKL
+#   GPU: SYCL
+#   Scope: configure, build, unit tests, regression tests
+# Test implementation choices (free to change as needed):
+#   OS: Ubuntu 20.04
+#   Build type: RelWithAssert
+#   Compiler: ICPX 2021.2
+#   MPI: thread_MPI
+#   SIMD: AVX2_256
+#   Parallelism nt/ntomp: 2/1
+
+gromacs:oneapi-2021.2.0-sycl-nightly:configure:
+  extends:
+   - .gromacs:base:configure
+   - .use-oneapi:base
+   - .use-sycl
+   - .rules:nightly-not-for-release
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+  variables:
+    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+    COMPILER_MAJOR_VERSION: 2021
+
+gromacs:oneapi-2021.2.0-sycl-nightly:build:
+  extends:
+    - .variables:default
+    - .gromacs:base:build
+    - .use-ccache
+    - .use-oneapi:base
+    - .rules:nightly-not-for-release
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+  variables:
+    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+  needs:
+    - job: gromacs:oneapi-2021.2.0-sycl-nightly:configure
+
+gromacs:oneapi-2021.2.0-sycl-nightly:test-opencl:
+  extends:
+    - .gromacs:base:test
+    - .use-oneapi:base
+    - .rules:nightly-not-for-release
+  tags:
+    - intel-gpu-dg1
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+  variables:
+    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+    SYCL_DEVICE_FILTER: "opencl:gpu"
+    # The following two variables are not used by k8s, since this job runs on a separate host
+    # But it's easier to set them up and use the existing scripts for job preparation that rely on them
+    KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu"
+    KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2
+    # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged
+    GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2
+  needs:
+    - job: gromacs:oneapi-2021.2.0-sycl-nightly:build
+
+gromacs:oneapi-2021.2.0-sycl-nightly:regressiontest-opencl:
+  extends:
+    - .gromacs:base:regressiontest
+    - .use-oneapi:base
+    - .rules:nightly-not-for-release
+  tags:
+    - intel-gpu-dg1
+  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+  variables:
+    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+    SYCL_DEVICE_FILTER: "opencl:gpu"
+    # Default not working, see #2980
+    REGRESSIONTEST_TOTAL_RANK_NUMBER: 2
+    REGRESSIONTEST_OMP_RANK_NUMBER: 1
+    # The following two variables are not used by k8s, since this job runs on a separate host
+    # But it's easier to set them up and use the existing scripts for job preparation that rely on them
+    KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu"
+    KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2
+    # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged
+    GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2
+  needs:
+    - job: gromacs:oneapi-2021.2.0-sycl-nightly:build
+    - job: regressiontests:prepare
+
+# We have to disable LevelZero tests until https://github.com/intel/llvm/pull/3045 makes it into
+# the OneAPI release. Without this patch, some of the tests deadlock.
+
+#gromacs:oneapi-2021.2.0-sycl-nightly:test-levelzero:
+#  extends:
+#    - .gromacs:base:test
+#    - .use-oneapi:base
+#    - .rules:nightly-not-for-release
+#  tags:
+#    - intel-gpu-dg1
+#  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+#  variables:
+#    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+#    SYCL_DEVICE_FILTER: "level_zero:gpu"
+#    # The following two variables are not used by k8s, since this job runs on a separate host
+#    # But it's easier to set them up and use the existing scripts for job preparation that rely on them
+#    KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu"
+#    KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2
+#    # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged
+#    GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2
+#  needs:
+#    - job: gromacs:oneapi-2021.2.0-sycl-nightly:build
+#
+#gromacs:oneapi-2021.2.0-sycl-nightly:regressiontest-levelzero:
+#  extends:
+#    - .gromacs:base:regressiontest
+#    - .use-oneapi:base
+#    - .rules:nightly-not-for-release
+#  tags:
+#    - intel-gpu-dg1
+#  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-7-oneapi-2021.2.0-intel-21.21.19914:latest
+#  variables:
+#    CMAKE: /usr/local/cmake-3.18.4/bin/cmake
+#    SYCL_DEVICE_FILTER: "level_zero:gpu"
+#    # Default not working, see #2980
+#    REGRESSIONTEST_TOTAL_RANK_NUMBER: 2
+#    REGRESSIONTEST_OMP_RANK_NUMBER: 1
+#    # The following two variables are not used by k8s, since this job runs on a separate host
+#    # But it's easier to set them up and use the existing scripts for job preparation that rely on them
+#    KUBERNETES_EXTENDED_RESOURCE_NAME: "intel.com/gpu"
+#    KUBERNETES_EXTENDED_RESOURCE_LIMIT: 2
+#    # Except we now don't request any GPUs, so we must set this one up manually until !1585 is merged
+#    GMX_TEST_REQUIRED_NUMBER_OF_DEVICES: 2
+#  needs:
+#    - job: gromacs:oneapi-2021.2.0-sycl-nightly:build
+#    - job: regressiontests:prepare