From f43126c31f8d2e694cb95b66fdf56add7830a178 Mon Sep 17 00:00:00 2001 From: Mark Abraham Date: Mon, 5 Nov 2018 23:28:44 +0100 Subject: [PATCH] Fix OpenCL mdrun from installed path Previously we could dump all the headers and source files in the share/gromacs/opencl directory, because they all came from the NB module. Now that we have code in the gpu_utils module used by both Ewald and NB modules, that is not an option. So we may as well use the same approach for the pbcutil and high-level NBNXN headers also. Change-Id: I894e6a8032547077bb5198cc6c40f65916d1b17f --- src/gromacs/CMakeLists.txt | 50 +++++++++++++++++-- src/gromacs/ewald/CMakeLists.txt | 2 - .../ewald/pme-gpu-program-impl-ocl.cpp | 2 +- src/gromacs/gpu_utils/CMakeLists.txt | 8 --- src/gromacs/gpu_utils/ocl_compiler.cpp | 14 +++--- src/gromacs/mdlib/nbnxn_ocl/CMakeLists.txt | 2 - .../mdlib/nbnxn_ocl/nbnxn_ocl_jit_support.cpp | 13 +++-- 7 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/gromacs/CMakeLists.txt b/src/gromacs/CMakeLists.txt index 9291b44451..3430faf749 100644 --- a/src/gromacs/CMakeLists.txt +++ b/src/gromacs/CMakeLists.txt @@ -383,8 +383,52 @@ if (INSTALL_CUDART_LIB) #can be set manual by user endif() if(GMX_USE_OPENCL) - set(OPENCL_KERNELS ${MDLIB_OPENCL_KERNELS}) + # Install the utility headers + file(GLOB OPENCL_INSTALLED_FILES + gpu_utils/vectype_ops.clh + gpu_utils/device_utils.clh + ) + install(FILES ${OPENCL_INSTALLED_FILES} + DESTINATION ${GMX_INSTALL_OCLDIR}/gromacs/gpu_utils + COMPONENT libraries) + file(GLOB OPENCL_INSTALLED_FILES + pbcutil/ishift.h + ) + install(FILES ${OPENCL_INSTALLED_FILES} + DESTINATION ${GMX_INSTALL_OCLDIR}/gromacs/pbcutil + COMPONENT libraries) - install(FILES ${OPENCL_KERNELS} DESTINATION - ${GMX_INSTALL_OCLDIR} COMPONENT libraries) + # Install the NB source and headers + file(GLOB OPENCL_INSTALLED_FILES + mdlib/nbnxn_consts.h + ) + install(FILES ${OPENCL_INSTALLED_FILES} + DESTINATION ${GMX_INSTALL_OCLDIR}/gromacs/mdlib + COMPONENT libraries) + file(GLOB OPENCL_INSTALLED_FILES + mdlib/nbnxn_ocl/nbnxn_ocl_kernels.cl + mdlib/nbnxn_ocl/nbnxn_ocl_kernel.clh + mdlib/nbnxn_ocl/nbnxn_ocl_kernel_pruneonly.clh + mdlib/nbnxn_ocl/nbnxn_ocl_kernels.clh + mdlib/nbnxn_ocl/nbnxn_ocl_kernels_fastgen.clh + mdlib/nbnxn_ocl/nbnxn_ocl_kernels_fastgen_add_twincut.clh + mdlib/nbnxn_ocl/nbnxn_ocl_kernel_utils.clh + mdlib/nbnxn_ocl/nbnxn_ocl_consts.h + ) + install(FILES ${OPENCL_INSTALLED_FILES} + DESTINATION ${GMX_INSTALL_OCLDIR}/gromacs/mdlib/nbnxn_ocl + COMPONENT libraries) + + # Install the PME source and headers + file(GLOB OPENCL_INSTALLED_FILES + ewald/pme-spread.clh + ewald/pme-solve.clh + ewald/pme-gather.clh + ewald/pme-gpu-utils.clh + ewald/pme-program.cl + ewald/pme-gpu-types.h + ) + install(FILES ${OPENCL_INSTALLED_FILES} + DESTINATION ${GMX_INSTALL_OCLDIR}/gromacs/ewald + COMPONENT libraries) endif() diff --git a/src/gromacs/ewald/CMakeLists.txt b/src/gromacs/ewald/CMakeLists.txt index aa916c2daf..05654eda2d 100644 --- a/src/gromacs/ewald/CMakeLists.txt +++ b/src/gromacs/ewald/CMakeLists.txt @@ -78,8 +78,6 @@ elseif (GMX_USE_OPENCL) pme-gpu-internal.cpp pme-gpu-timings.cpp ) - file(GLOB EWALD_OPENCL_RUNTIME_SOURCES pme-spread.clh pme-solve.clh pme-gather.clh pme-gpu-utils.clh pme-program.cl pme-gpu-types.h) - set(MDLIB_OPENCL_KERNELS ${MDLIB_OPENCL_KERNELS} ${EWALD_OPENCL_RUNTIME_SOURCES} PARENT_SCOPE) else() gmx_add_libgromacs_sources( # Files that implement stubs diff --git a/src/gromacs/ewald/pme-gpu-program-impl-ocl.cpp b/src/gromacs/ewald/pme-gpu-program-impl-ocl.cpp index 43b533651d..16bfac9aad 100644 --- a/src/gromacs/ewald/pme-gpu-program-impl-ocl.cpp +++ b/src/gromacs/ewald/pme-gpu-program-impl-ocl.cpp @@ -176,7 +176,7 @@ void PmeGpuProgramImpl::compileKernels(const gmx_device_info_t *deviceInfo) /* TODO when we have a proper MPI-aware logging module, the log output here should be written there */ program = gmx::ocl::compileProgram(stderr, - "src/gromacs/ewald", + "gromacs/ewald", "pme-program.cl", commonDefines, context, diff --git a/src/gromacs/gpu_utils/CMakeLists.txt b/src/gromacs/gpu_utils/CMakeLists.txt index 6f2c5bc412..2037543679 100644 --- a/src/gromacs/gpu_utils/CMakeLists.txt +++ b/src/gromacs/gpu_utils/CMakeLists.txt @@ -57,14 +57,6 @@ elseif(GMX_USE_CUDA) ) endif() -if(GMX_USE_OPENCL) - file(GLOB OPENCL_COMMON_HEADERS - vectype_ops.clh - device_utils.clh - ) - set(MDLIB_OPENCL_KERNELS ${MDLIB_OPENCL_KERNELS} ${OPENCL_COMMON_HEADERS} PARENT_SCOPE) -endif() - # workaround for the state - host allocator dependency gmx_install_headers(hostallocator.h) diff --git a/src/gromacs/gpu_utils/ocl_compiler.cpp b/src/gromacs/gpu_utils/ocl_compiler.cpp index 4e8866923d..f5b093e974 100644 --- a/src/gromacs/gpu_utils/ocl_compiler.cpp +++ b/src/gromacs/gpu_utils/ocl_compiler.cpp @@ -216,7 +216,7 @@ selectCompilerOptions(ocl_vendor_id_t deviceVendorId) * behavior by defining GMX_OCL_FILE_PATH environment variable. * * \param[in] sourceRelativePath Relative path to the kernel or other file in the source tree, - * e.g. "src/gromacs/mdlib/nbnxn_ocl" for NB kernels. + * from src, e.g. "gromacs/mdlib/nbnxn_ocl" for NB kernels. * \return OS-normalized path string to the folder storing OpenCL source file * * \throws std::bad_alloc if out of memory. @@ -235,9 +235,9 @@ getSourceRootPath(const std::string &sourceRelativePath) root path from the path to the binary that is running. */ InstallationPrefixInfo info = getProgramContext().installationPrefix(); std::string dataPathSuffix = (info.bSourceLayout ? - sourceRelativePath : + "src" : GMX_INSTALL_OCLDIR); - sourceRootPath = Path::join(info.path, dataPathSuffix); + sourceRootPath = Path::join(info.path, dataPathSuffix, sourceRelativePath); } else { @@ -246,7 +246,7 @@ getSourceRootPath(const std::string &sourceRelativePath) GMX_THROW(FileIOError(formatString("GMX_OCL_FILE_PATH must point to the directory where OpenCL" "kernels are found, but '%s' does not exist", gmxOclFilePath))); } - sourceRootPath = gmxOclFilePath; + sourceRootPath = Path::join(gmxOclFilePath, sourceRelativePath); } // Make sure we return an OS-correct path format @@ -425,8 +425,10 @@ compileProgram(FILE *fplog, ocl_vendor_id_t deviceVendorId) { cl_int cl_error; + // Let the kernel find include files from its module. std::string kernelRootPath = getSourceRootPath(kernelRelativePath); - std::string includeRootPath = getSourceRootPath("src"); + // Let the kernel find include files from other modules. + std::string rootPath = getSourceRootPath(""); GMX_RELEASE_ASSERT(fplog != nullptr, "Need a valid log file for building OpenCL programs"); @@ -436,7 +438,7 @@ compileProgram(FILE *fplog, /* Make the build options */ std::string preprocessorOptions = makePreprocessorOptions(kernelRootPath, - includeRootPath, + rootPath, getDeviceWarpSize(context, deviceId), deviceVendorId, extraDefines); diff --git a/src/gromacs/mdlib/nbnxn_ocl/CMakeLists.txt b/src/gromacs/mdlib/nbnxn_ocl/CMakeLists.txt index 816928b317..0e837390c0 100644 --- a/src/gromacs/mdlib/nbnxn_ocl/CMakeLists.txt +++ b/src/gromacs/mdlib/nbnxn_ocl/CMakeLists.txt @@ -35,8 +35,6 @@ if(GMX_USE_OPENCL) file(GLOB OPENCL_NB_SOURCES *.cpp) set(MDLIB_SOURCES ${MDLIB_SOURCES} ${OPENCL_NB_SOURCES} PARENT_SCOPE) - file(GLOB MDLIB_OPENCL_KERNELS *.cl *.clh) - set(MDLIB_OPENCL_KERNELS ${MDLIB_OPENCL_KERNELS} PARENT_SCOPE) endif() set(ELEC_DEFS diff --git a/src/gromacs/mdlib/nbnxn_ocl/nbnxn_ocl_jit_support.cpp b/src/gromacs/mdlib/nbnxn_ocl/nbnxn_ocl_jit_support.cpp index 155cd32589..14328f0b74 100644 --- a/src/gromacs/mdlib/nbnxn_ocl/nbnxn_ocl_jit_support.cpp +++ b/src/gromacs/mdlib/nbnxn_ocl/nbnxn_ocl_jit_support.cpp @@ -192,24 +192,23 @@ nbnxn_gpu_compile_kernels(gmx_nbnxn_ocl_t *nb) nb->nbparam->eeltype, nb->nbparam->vdwtype); - /* Here we pass macros and static const int variables defined in include - * files outside the nbnxn_ocl as macros, to avoid including those files - * in the JIT compilation that happens at runtime. - */ - + /* Here we pass macros and static const int variables defined + * in include files outside the nbnxn_ocl as macros, to avoid + * including those files in the JIT compilation that happens + * at runtime. This is particularly a problem for headers that + * depend on config.h, such as nbnxn_pairlist.h. */ extraDefines += gmx::formatString( " -DNBNXN_GPU_CLUSTER_SIZE=%d " "%s", c_nbnxnGpuClusterSize, /* Defined in nbnxn_pairlist.h */ (nb->bPrefetchLjParam) ? "-DIATYPE_SHMEM" : "" ); - try { /* TODO when we have a proper MPI-aware logging module, the log output here should be written there */ program = gmx::ocl::compileProgram(stderr, - "src/gromacs/mdlib/nbnxn_ocl", + "gromacs/mdlib/nbnxn_ocl", "nbnxn_ocl_kernels.cl", extraDefines, nb->dev_rundata->context, -- 2.22.0