src/gromacs/ewald/pme_gpu_program_impl_ocl.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35
  36 /*! \internal \file
  37  * \brief
  38  * Implements PmeGpuProgramImpl, which stores permanent PME GPU context-derived data,
  39  * such as (compiled) kernel handles.
  40  *
  41  * \author Aleksei Iupinov <a.yupinov@gmail.com>
  42  * \ingroup module_ewald
  43  */
  44 #include "gmxpre.h"
  45
  46 #include "gromacs/gpu_utils/gmxopencl.h"
  47 #include "gromacs/gpu_utils/ocl_compiler.h"
  48 #include "gromacs/utility/stringutil.h"
  49
  50 #include "pme_gpu_constants.h"
  51 #include "pme_gpu_internal.h" // for GridOrdering enum
  52 #include "pme_gpu_program_impl.h"
  53 #include "pme_gpu_types_host.h"
  54 #include "pme_grid.h"
  55
  56 PmeGpuProgramImpl::PmeGpuProgramImpl(const DeviceContext& deviceContext) :
  57     deviceContext_(deviceContext)
  58 {
  59     const DeviceInformation& deviceInfo = deviceContext.deviceInfo();
  60     // kernel parameters
  61     warpSize_ = gmx::ocl::getDeviceWarpSize(deviceContext_.context(), deviceInfo.oclDeviceId);
  62     // TODO: for Intel ideally we'd want to set these based on the compiler warp size
  63     // but given that we've done no tuning for Intel iGPU, this is as good as anything.
  64     spreadWorkGroupSize = std::min(c_spreadMaxWarpsPerBlock * warpSize_, deviceInfo.maxWorkGroupSize);
  65     solveMaxWorkGroupSize = std::min(c_solveMaxWarpsPerBlock * warpSize_, deviceInfo.maxWorkGroupSize);
  66     gatherWorkGroupSize = std::min(c_gatherMaxWarpsPerBlock * warpSize_, deviceInfo.maxWorkGroupSize);
  67
  68     compileKernels(deviceInfo);
  69 }
  70
  71 PmeGpuProgramImpl::~PmeGpuProgramImpl()
  72 {
  73     // TODO: log releasing errors
  74     cl_int gmx_used_in_debug stat = 0;
  75     stat |= clReleaseKernel(splineAndSpreadKernelSingle);
  76     stat |= clReleaseKernel(splineKernelSingle);
  77     stat |= clReleaseKernel(spreadKernelSingle);
  78     stat |= clReleaseKernel(splineAndSpreadKernelDual);
  79     stat |= clReleaseKernel(splineKernelDual);
  80     stat |= clReleaseKernel(spreadKernelDual);
  81     stat |= clReleaseKernel(gatherKernelSingle);
  82     stat |= clReleaseKernel(gatherKernelDual);
  83     stat |= clReleaseKernel(solveXYZKernelA);
  84     stat |= clReleaseKernel(solveXYZEnergyKernelA);
  85     stat |= clReleaseKernel(solveYZXKernelA);
  86     stat |= clReleaseKernel(solveYZXEnergyKernelA);
  87     stat |= clReleaseKernel(solveXYZKernelB);
  88     stat |= clReleaseKernel(solveXYZEnergyKernelB);
  89     stat |= clReleaseKernel(solveYZXKernelB);
  90     stat |= clReleaseKernel(solveYZXEnergyKernelB);
  91     GMX_ASSERT(stat == CL_SUCCESS,
  92                gmx::formatString("Failed to release PME OpenCL resources %d: %s",
  93                                  stat,
  94                                  ocl_get_error_string(stat).c_str())
  95                        .c_str());
  96 }
  97
  98 /*! \brief Ensure that spread/gather kernels have been compiled to a suitable warp size
  99  *
 100  * On Intel the exec width/warp is decided at compile-time and can be
 101  * smaller than the minimum order^2 required in spread/gather ATM which
 102  * we need to check for.
 103  *
 104  * Due to the one thread per atom and order=4 implementation
 105  * constraints, order^2 threads should execute without synchronization
 106  * needed.
 107  */
 108 static void checkRequiredWarpSize(cl_kernel kernel, const char* kernelName, const DeviceInformation& deviceInfo)
 109 {
 110     if (deviceInfo.deviceVendor == DeviceVendor::Intel)
 111     {
 112         int       kernelWarpSize    = gmx::ocl::getKernelWarpSize(kernel, deviceInfo.oclDeviceId);
 113         const int minKernelWarpSize = c_pmeGpuOrder * c_pmeGpuOrder;
 114         if (kernelWarpSize < minKernelWarpSize)
 115         {
 116             const std::string errorString = gmx::formatString(
 117                     "PME OpenCL kernels require >=%d execution width, but the %s kernel "
 118                     "has been compiled for the device %s to a %d width and therefore it can not "
 119                     "execute correctly.",
 120                     minKernelWarpSize,
 121                     kernelName,
 122                     deviceInfo.device_name,
 123                     kernelWarpSize);
 124             GMX_THROW(gmx::InternalError(errorString));
 125         }
 126     }
 127 }
 128
 129 void PmeGpuProgramImpl::compileKernels(const DeviceInformation& deviceInfo)
 130 {
 131     // We might consider storing program as a member variable if it's needed later
 132     cl_program program = nullptr;
 133     /* Need to catch std::bad_alloc here and during compilation string handling. */
 134     try
 135     {
 136         /* Here we pass macros and static const int variables defined in include
 137          * files outside as macros, to avoid including those files
 138          * in the JIT compilation that happens at runtime.
 139          */
 140         const std::string commonDefines = gmx::formatString(
 141                 "-Dwarp_size=%zd "
 142                 "-Dorder=%d "
 143                 "-DthreadsPerAtom=%d "
 144                 // forwarding from pme_grid.h, used for spline computation table sizes only
 145                 "-Dc_pmeMaxUnitcellShift=%f "
 146                 // forwarding PME behavior constants from pme_gpu_constants.h
 147                 "-Dc_skipNeutralAtoms=%d "
 148                 "-Dc_virialAndEnergyCount=%d "
 149                 // forwarding kernel work sizes
 150                 "-Dc_spreadWorkGroupSize=%zd "
 151                 "-Dc_solveMaxWorkGroupSize=%zd "
 152                 "-Dc_gatherWorkGroupSize=%zd "
 153                 // forwarding from vectypes.h
 154                 "-DDIM=%d -DXX=%d -DYY=%d -DZZ=%d "
 155                 // decomposition parameter placeholders
 156                 "-DwrapX=true -DwrapY=true ",
 157                 warpSize_,
 158                 c_pmeGpuOrder,
 159                 c_pmeGpuOrder * c_pmeGpuOrder,
 160                 static_cast<float>(c_pmeMaxUnitcellShift),
 161                 static_cast<int>(c_skipNeutralAtoms),
 162                 c_virialAndEnergyCount,
 163                 spreadWorkGroupSize,
 164                 solveMaxWorkGroupSize,
 165                 gatherWorkGroupSize,
 166                 DIM,
 167                 XX,
 168                 YY,
 169                 ZZ);
 170         try
 171         {
 172             /* TODO when we have a proper MPI-aware logging module,
 173                the log output here should be written there */
 174             program = gmx::ocl::compileProgram(stderr,
 175                                                "gromacs/ewald",
 176                                                "pme_program.cl",
 177                                                commonDefines,
 178                                                deviceContext_.context(),
 179                                                deviceInfo.oclDeviceId,
 180                                                deviceInfo.deviceVendor);
 181         }
 182         catch (gmx::GromacsException& e)
 183         {
 184             e.prependContext(gmx::formatString("Failed to compile PME kernels for GPU #%s\n",
 185                                                deviceInfo.device_name));
 186             throw;
 187         }
 188     }
 189     GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
 190
 191     constexpr cl_uint expectedKernelCount = 18;
 192     // Has to be equal or larger than the number of kernel instances.
 193     // If it is not, CL_INVALID_VALUE will be thrown.
 194     std::vector<cl_kernel> kernels(expectedKernelCount, nullptr);
 195     cl_uint                actualKernelCount = 0;
 196     cl_int clError = clCreateKernelsInProgram(program, kernels.size(), kernels.data(), &actualKernelCount);
 197     if (clError != CL_SUCCESS)
 198     {
 199         const std::string errorString = gmx::formatString(
 200                 "Failed to create kernels for PME on GPU #%s:\n OpenCL error %d: %s",
 201                 deviceInfo.device_name,
 202                 clError,
 203                 ocl_get_error_string(clError).c_str());
 204         GMX_THROW(gmx::InternalError(errorString));
 205     }
 206     kernels.resize(actualKernelCount);
 207
 208     std::array<char, 100> kernelNamesBuffer;
 209     for (const auto& kernel : kernels)
 210     {
 211         clError = clGetKernelInfo(
 212                 kernel, CL_KERNEL_FUNCTION_NAME, kernelNamesBuffer.size(), kernelNamesBuffer.data(), nullptr);
 213         if (clError != CL_SUCCESS)
 214         {
 215             const std::string errorString = gmx::formatString(
 216                     "Failed to parse kernels for PME on GPU #%s:\n OpenCL error %d: %s",
 217                     deviceInfo.device_name,
 218                     clError,
 219                     ocl_get_error_string(clError).c_str());
 220             GMX_THROW(gmx::InternalError(errorString));
 221         }
 222
 223         // The names below must correspond to those defined in pme_program.cl
 224         // TODO use a map with string key instead?
 225         if (!strcmp(kernelNamesBuffer.data(), "pmeSplineKernelSingle"))
 226         {
 227             splineKernelSingle = kernel;
 228         }
 229         else if (!strcmp(kernelNamesBuffer.data(), "pmeSplineAndSpreadKernelSingle"))
 230         {
 231             splineAndSpreadKernelSingle             = kernel;
 232             splineAndSpreadKernelWriteSplinesSingle = kernel;
 233             checkRequiredWarpSize(splineAndSpreadKernelSingle, kernelNamesBuffer.data(), deviceInfo);
 234         }
 235         else if (!strcmp(kernelNamesBuffer.data(), "pmeSpreadKernelSingle"))
 236         {
 237             spreadKernelSingle = kernel;
 238             checkRequiredWarpSize(spreadKernelSingle, kernelNamesBuffer.data(), deviceInfo);
 239         }
 240         else if (!strcmp(kernelNamesBuffer.data(), "pmeSplineKernelDual"))
 241         {
 242             splineKernelDual = kernel;
 243         }
 244         else if (!strcmp(kernelNamesBuffer.data(), "pmeSplineAndSpreadKernelDual"))
 245         {
 246             splineAndSpreadKernelDual             = kernel;
 247             splineAndSpreadKernelWriteSplinesDual = kernel;
 248             checkRequiredWarpSize(splineAndSpreadKernelDual, kernelNamesBuffer.data(), deviceInfo);
 249         }
 250         else if (!strcmp(kernelNamesBuffer.data(), "pmeSpreadKernelDual"))
 251         {
 252             spreadKernelDual = kernel;
 253             checkRequiredWarpSize(spreadKernelDual, kernelNamesBuffer.data(), deviceInfo);
 254         }
 255         else if (!strcmp(kernelNamesBuffer.data(), "pmeGatherKernelSingle"))
 256         {
 257             gatherKernelSingle            = kernel;
 258             gatherKernelReadSplinesSingle = kernel;
 259             checkRequiredWarpSize(gatherKernelSingle, kernelNamesBuffer.data(), deviceInfo);
 260         }
 261         else if (!strcmp(kernelNamesBuffer.data(), "pmeGatherKernelDual"))
 262         {
 263             gatherKernelDual            = kernel;
 264             gatherKernelReadSplinesDual = kernel;
 265             checkRequiredWarpSize(gatherKernelDual, kernelNamesBuffer.data(), deviceInfo);
 266         }
 267         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveYZXKernelA"))
 268         {
 269             solveYZXKernelA = kernel;
 270         }
 271         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveYZXEnergyKernelA"))
 272         {
 273             solveYZXEnergyKernelA = kernel;
 274         }
 275         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveXYZKernelA"))
 276         {
 277             solveXYZKernelA = kernel;
 278         }
 279         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveXYZEnergyKernelA"))
 280         {
 281             solveXYZEnergyKernelA = kernel;
 282         }
 283         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveYZXKernelB"))
 284         {
 285             solveYZXKernelB = kernel;
 286         }
 287         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveYZXEnergyKernelB"))
 288         {
 289             solveYZXEnergyKernelB = kernel;
 290         }
 291         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveXYZKernelB"))
 292         {
 293             solveXYZKernelB = kernel;
 294         }
 295         else if (!strcmp(kernelNamesBuffer.data(), "pmeSolveXYZEnergyKernelB"))
 296         {
 297             solveXYZEnergyKernelB = kernel;
 298         }
 299     }
 300     clReleaseProgram(program);
 301 }