2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2014,2015,2016,2017,2018 by the GROMACS development team.
5 * Copyright (c) 2019,2020, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
36 /*! \libinternal \file
37 * \brief Declare utility routines for OpenCL
39 * \author Anca Hamuraru <anca@streamcomputing.eu>
42 #ifndef GMX_GPU_UTILS_OCLUTILS_H
43 #define GMX_GPU_UTILS_OCLUTILS_H
47 #include "gromacs/gpu_utils/device_context.h"
48 #include "gromacs/gpu_utils/device_stream.h"
49 #include "gromacs/gpu_utils/gmxopencl.h"
50 #include "gromacs/gpu_utils/gputraits_ocl.h"
51 #include "gromacs/utility/exceptions.h"
52 #include "gromacs/utility/gmxassert.h"
54 enum class GpuApiCallBehavior;
57 * \brief OpenCL GPU runtime data
59 * The device runtime data is meant to hold objects associated with a GROMACS rank's
60 * (thread or process) use of a single device (multiple devices per rank is not
61 * implemented). These objects should be constructed at ther point where a device
62 * dets assigned to a rank and released at when this assignment is no longer valid
63 * (i.e. at cleanup in the current implementation).
66 struct gmx_device_runtime_data_t
69 gmx_device_runtime_data_t(const DeviceContext& deviceContext) : deviceContext_(deviceContext) {}
72 const DeviceContext& deviceContext_;
77 /*! \brief Launches synchronous or asynchronous device to host memory copy.
79 * If copy_event is not NULL, on return it will contain an event object
80 * identifying this particular device to host operation. The event can further
81 * be used to queue a wait for this operation or to query profiling information.
83 int ocl_copy_D2H(void* h_dest,
87 GpuApiCallBehavior transferKind,
88 cl_command_queue command_queue,
89 cl_event* copy_event);
92 /*! \brief Launches asynchronous device to host memory copy. */
93 int ocl_copy_D2H_async(void* h_dest,
97 cl_command_queue command_queue,
98 cl_event* copy_event);
100 /*! \brief Launches synchronous or asynchronous host to device memory copy.
102 * If copy_event is not NULL, on return it will contain an event object
103 * identifying this particular host to device operation. The event can further
104 * be used to queue a wait for this operation or to query profiling information.
106 int ocl_copy_H2D(cl_mem d_dest,
110 GpuApiCallBehavior transferKind,
111 cl_command_queue command_queue,
112 cl_event* copy_event);
114 /*! \brief Launches asynchronous host to device memory copy. */
115 int ocl_copy_H2D_async(cl_mem d_dest,
119 cl_command_queue command_queue,
120 cl_event* copy_event);
122 /*! \brief Launches synchronous host to device memory copy. */
123 int ocl_copy_H2D_sync(cl_mem d_dest, const void* h_src, size_t offset, size_t bytes, cl_command_queue command_queue);
125 /*! \brief Allocate host memory in malloc style */
126 void pmalloc(void** h_ptr, size_t nbytes);
128 /*! \brief Free host memory in malloc style */
129 void pfree(void* h_ptr);
131 /*! \brief Convert error code to diagnostic string */
132 std::string ocl_get_error_string(cl_int error);
134 //! A debug checker to track cl_events being released correctly
135 inline void ensureReferenceCount(const cl_event& event, unsigned int refCount)
138 cl_int clError = clGetEventInfo(event, CL_EVENT_REFERENCE_COUNT, sizeof(refCount), &refCount, nullptr);
139 GMX_ASSERT(CL_SUCCESS == clError, ocl_get_error_string(clError).c_str());
140 GMX_ASSERT(refCount == refCount, "Unexpected reference count");
142 GMX_UNUSED_VALUE(event);
143 GMX_UNUSED_VALUE(refCount);
147 /*! \brief Pretend to synchronize an OpenCL stream (dummy implementation).
149 * \returns Not implemented in OpenCL.
151 static inline bool haveStreamTasksCompleted(const DeviceStream& /* deviceStream */)
153 GMX_RELEASE_ASSERT(false, "haveStreamTasksCompleted is not implemented for OpenCL");
157 /* Kernel launch helpers */
160 * A function for setting up a single OpenCL kernel argument.
161 * This is the tail of the compile-time recursive function below.
162 * It has to be seen by the compiler first.
163 * As NB kernels might be using dynamic local memory as the last argument,
164 * this function also manages that, using sharedMemorySize from \p config.
166 * \param[in] kernel Kernel function handle
167 * \param[in] config Kernel configuration for launching
168 * \param[in] argIndex Index of the current argument
170 void inline prepareGpuKernelArgument(cl_kernel kernel, const KernelLaunchConfig& config, size_t argIndex)
172 if (config.sharedMemorySize > 0)
174 cl_int gmx_used_in_debug clError =
175 clSetKernelArg(kernel, argIndex, config.sharedMemorySize, nullptr);
176 GMX_ASSERT(CL_SUCCESS == clError, ocl_get_error_string(clError).c_str());
181 * Compile-time recursive function for setting up a single OpenCL kernel argument.
182 * This function uses one kernel argument pointer \p argPtr to call clSetKernelArg(),
183 * and calls itself on the next argument, eventually calling the tail function above.
185 * \tparam CurrentArg Type of the current argument
186 * \tparam RemainingArgs Types of remaining arguments after the current one
187 * \param[in] kernel Kernel function handle
188 * \param[in] config Kernel configuration for launching
189 * \param[in] argIndex Index of the current argument
190 * \param[in] argPtr Pointer to the current argument
191 * \param[in] otherArgsPtrs Pack of pointers to arguments remaining to process after the current one
193 template<typename CurrentArg, typename... RemainingArgs>
194 void prepareGpuKernelArgument(cl_kernel kernel,
195 const KernelLaunchConfig& config,
197 const CurrentArg* argPtr,
198 const RemainingArgs*... otherArgsPtrs)
200 cl_int gmx_used_in_debug clError = clSetKernelArg(kernel, argIndex, sizeof(CurrentArg), argPtr);
201 GMX_ASSERT(CL_SUCCESS == clError, ocl_get_error_string(clError).c_str());
203 // Assert on types not allowed to be passed to a kernel
204 // (as per section 6.9 of the OpenCL spec).
205 static_assert(!std::is_same<CurrentArg, bool>::value && !std::is_same<CurrentArg, size_t>::value
206 && !std::is_same<CurrentArg, ptrdiff_t>::value
207 && !std::is_same<CurrentArg, intptr_t>::value
208 && !std::is_same<CurrentArg, uintptr_t>::value,
209 "Invalid type passed to OpenCL kernel functions (see OpenCL spec section 6.9).");
211 prepareGpuKernelArgument(kernel, config, argIndex + 1, otherArgsPtrs...);
215 * A wrapper function for setting up all the OpenCL kernel arguments.
216 * Calls the recursive functions above.
218 * \tparam Args Types of all the kernel arguments
219 * \param[in] kernel Kernel function handle
220 * \param[in] config Kernel configuration for launching
221 * \param[in] argsPtrs Pointers to all the kernel arguments
222 * \returns A handle for the prepared parameter pack to be used with launchGpuKernel() as the last argument
223 * - currently always nullptr for OpenCL, as it manages kernel/arguments association by itself.
225 template<typename... Args>
226 void* prepareGpuKernelArguments(cl_kernel kernel, const KernelLaunchConfig& config, const Args*... argsPtrs)
228 prepareGpuKernelArgument(kernel, config, 0, argsPtrs...);
232 /*! \brief Launches the OpenCL kernel and handles the errors.
234 * \param[in] kernel Kernel function handle
235 * \param[in] config Kernel configuration for launching
236 * \param[in] timingEvent Timing event, fetched from GpuRegionTimer
237 * \param[in] kernelName Human readable kernel description, for error handling only
238 * \throws gmx::InternalError on kernel launch failure
240 inline void launchGpuKernel(cl_kernel kernel,
241 const KernelLaunchConfig& config,
242 CommandEvent* timingEvent,
243 const char* kernelName,
244 const void* /*kernelArgs*/)
246 const int workDimensions = 3;
247 const size_t* globalWorkOffset = nullptr;
248 const size_t waitListSize = 0;
249 const cl_event* waitList = nullptr;
250 size_t globalWorkSize[3];
251 for (int i = 0; i < workDimensions; i++)
253 globalWorkSize[i] = config.gridSize[i] * config.blockSize[i];
255 cl_int clError = clEnqueueNDRangeKernel(config.stream, kernel, workDimensions, globalWorkOffset,
256 globalWorkSize, config.blockSize, waitListSize,
257 waitList, timingEvent);
258 if (CL_SUCCESS != clError)
260 const std::string errorMessage = "GPU kernel (" + std::string(kernelName)
261 + ") failed to launch: " + ocl_get_error_string(clError);
262 GMX_THROW(gmx::InternalError(errorMessage));