2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015,2017,2018,2019, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
36 * \brief Declare interface for GPU execution for NBNXN module
38 * \author Szilard Pall <pall.szilard@gmail.com>
39 * \author Mark Abraham <mark.j.abraham@gmail.com>
40 * \ingroup module_nbnxm
43 #ifndef GMX_NBNXM_NBNXM_GPU_H
44 #define GMX_NBNXM_NBNXM_GPU_H
46 #include "gromacs/gpu_utils/gpu_macros.h"
47 #include "gromacs/math/vectypes.h"
48 #include "gromacs/nbnxm/atomdata.h"
49 #include "gromacs/utility/basedefinitions.h"
50 #include "gromacs/utility/real.h"
52 #include "gpu_types.h"
55 struct nbnxn_atomdata_t;
57 enum class GpuTaskCompletion;
71 * Launch asynchronously the xq buffer host to device copy.
73 * The nonlocal copy is skipped if there is no dependent work to do,
74 * neither non-local nonbonded interactions nor bonded GPU work.
76 * \param [in] nb GPU nonbonded data.
77 * \param [in] nbdata Host-side atom data structure.
78 * \param [in] aloc Atom locality flag.
81 void gpu_copy_xq_to_gpu(gmx_nbnxn_gpu_t gmx_unused *nb,
82 const struct nbnxn_atomdata_t gmx_unused *nbdata,
83 AtomLocality gmx_unused aloc) GPU_FUNC_TERM;
86 * Launch asynchronously the nonbonded force calculations.
88 * Also launches the initial pruning of a fresh list after search.
90 * The local and non-local interaction calculations are launched in two
91 * separate streams. If there is no work (i.e. empty pair list), the
92 * force kernel launch is omitted.
96 void gpu_launch_kernel(gmx_nbnxn_gpu_t gmx_unused *nb,
97 const gmx::ForceFlags gmx_unused &forceFlags,
98 InteractionLocality gmx_unused iloc) GPU_FUNC_TERM;
101 * Launch asynchronously the nonbonded prune-only kernel.
103 * The local and non-local list pruning are launched in their separate streams.
105 * Notes for future scheduling tuning:
106 * Currently we schedule the dynamic pruning between two MD steps *after* both local and
107 * nonlocal force D2H transfers completed. We could launch already after the cpyback
108 * is launched, but we want to avoid prune kernels (especially in the non-local
109 * high prio-stream) competing with nonbonded work.
111 * However, this is not ideal as this schedule does not expose the available
112 * concurrency. The dynamic pruning kernel:
113 * - should be allowed to overlap with any task other than force compute, including
114 * transfers (F D2H and the next step's x H2D as well as force clearing).
115 * - we'd prefer to avoid competition with non-bonded force kernels belonging
116 * to the same rank and ideally other ranks too.
118 * In the most general case, the former would require scheduling pruning in a separate
119 * stream and adding additional event sync points to ensure that force kernels read
120 * consistent pair list data. This would lead to some overhead (due to extra
121 * cudaStreamWaitEvent calls, 3-5 us/call) which we might be able to live with.
122 * The gains from additional overlap might not be significant as long as
123 * update+constraints anyway takes longer than pruning, but there will still
124 * be use-cases where more overlap may help (e.g. multiple ranks per GPU,
125 * no/hbonds only constraints).
126 * The above second point is harder to address given that multiple ranks will often
127 * share a GPU. Ranks that complete their nonbondeds sooner can schedule pruning earlier
128 * and without a third priority level it is difficult to avoid some interference of
129 * prune kernels with force tasks (in particular preemption of low-prio local force task).
131 * \param [inout] nb GPU nonbonded data.
132 * \param [in] iloc Interaction locality flag.
133 * \param [in] numParts Number of parts the pair list is split into in the rolling kernel.
136 void gpu_launch_kernel_pruneonly(gmx_nbnxn_gpu_t gmx_unused *nb,
137 InteractionLocality gmx_unused iloc,
138 int gmx_unused numParts) GPU_FUNC_TERM;
141 * Launch asynchronously the download of short-range forces from the GPU
142 * (and energies/shift forces if required).
145 void gpu_launch_cpyback(gmx_nbnxn_gpu_t gmx_unused *nb,
146 nbnxn_atomdata_t gmx_unused *nbatom,
147 const gmx::ForceFlags gmx_unused &forceFlags,
148 AtomLocality gmx_unused aloc,
149 bool gmx_unused copyBackNbForce) GPU_FUNC_TERM;
151 /*! \brief Attempts to complete nonbonded GPU task.
153 * This function attempts to complete the nonbonded task (both GPU and CPU auxiliary work).
154 * Success, i.e. that the tasks completed and results are ready to be consumed, is signaled
155 * by the return value (always true if blocking wait mode requested).
157 * The \p completionKind parameter controls whether the behavior is non-blocking
158 * (achieved by passing GpuTaskCompletion::Check) or blocking wait until the results
159 * are ready (when GpuTaskCompletion::Wait is passed).
160 * As the "Check" mode the function will return immediately if the GPU stream
161 * still contain tasks that have not completed, it allows more flexible overlapping
162 * of work on the CPU with GPU execution.
164 * Note that it is only safe to use the results, and to continue to the next MD
165 * step when this function has returned true which indicates successful completion of
166 * - All nonbonded GPU tasks: both compute and device transfer(s)
167 * - auxiliary tasks: updating the internal module state (timing accumulation, list pruning states) and
168 * - internal staging reduction of (\p fshift, \p e_el, \p e_lj).
170 * In GpuTaskCompletion::Check mode this function does the timing and keeps correct count
171 * for the nonbonded task (incrementing only once per taks), in the GpuTaskCompletion::Wait mode
172 * timing is expected to be done in the caller.
174 * TODO: improve the handling of outputs e.g. by ensuring that this function explcitly returns the
175 * force buffer (instead of that being passed only to nbnxn_gpu_launch_cpyback()) and by returning
176 * the energy and Fshift contributions for some external/centralized reduction.
178 * \param[in] nb The nonbonded data GPU structure
179 * \param[in] forceFlags Force schedule flags
180 * \param[in] aloc Atom locality identifier
181 * \param[out] e_lj Pointer to the LJ energy output to accumulate into
182 * \param[out] e_el Pointer to the electrostatics energy output to accumulate into
183 * \param[out] shiftForces Shift forces buffer to accumulate into
184 * \param[in] completionKind Indicates whether nnbonded task completion should only be checked rather than waited for
185 * \param[out] wcycle Pointer to wallcycle data structure
186 * \returns True if the nonbonded tasks associated with \p aloc locality have completed
189 bool gpu_try_finish_task(gmx_nbnxn_gpu_t gmx_unused *nb,
190 const gmx::ForceFlags gmx_unused &forceFlags,
191 AtomLocality gmx_unused aloc,
192 real gmx_unused *e_lj,
193 real gmx_unused *e_el,
194 gmx::ArrayRef<gmx::RVec> gmx_unused shiftForces,
195 GpuTaskCompletion gmx_unused completionKind,
196 gmx_wallcycle gmx_unused *wcycle) GPU_FUNC_TERM_WITH_RETURN(false);
198 /*! \brief Completes the nonbonded GPU task blocking until GPU tasks and data
199 * transfers to finish.
201 * Also does timing accounting and reduction of the internal staging buffers.
202 * As this is called at the end of the step, it also resets the pair list and
205 * \param[in] nb The nonbonded data GPU structure
206 * \param[in] forceFlags Force schedule flags
207 * \param[in] aloc Atom locality identifier
208 * \param[out] e_lj Pointer to the LJ energy output to accumulate into
209 * \param[out] e_el Pointer to the electrostatics energy output to accumulate into
210 * \param[out] shiftForces Shift forces buffer to accumulate into
213 float gpu_wait_finish_task(gmx_nbnxn_gpu_t gmx_unused *nb,
214 const gmx::ForceFlags gmx_unused &forceFlags,
215 AtomLocality gmx_unused aloc,
216 real gmx_unused *e_lj,
217 real gmx_unused *e_el,
218 gmx::ArrayRef<gmx::RVec> gmx_unused shiftForces,
219 gmx_wallcycle gmx_unused *wcycle) GPU_FUNC_TERM_WITH_RETURN(0.0);
221 /*! \brief Selects the Ewald kernel type, analytical or tabulated, single or twin cut-off. */
223 int gpu_pick_ewald_kernel_type(bool gmx_unused bTwinCut) GPU_FUNC_TERM_WITH_RETURN(-1);
225 /*! \brief Initialization for X buffer operations on GPU.
226 * Called on the NS step and performs (re-)allocations and memory copies. !*/
228 void nbnxn_gpu_init_x_to_nbat_x(const Nbnxm::GridSet gmx_unused &gridSet,
229 gmx_nbnxn_gpu_t gmx_unused *gpu_nbv) CUDA_FUNC_TERM;
231 /*! \brief Copy coordinates from host to device memory.
233 * \todo This will be removed as the management of the buffers is taken out of the NBNXM module.
235 * \param[in] grid Grid to be copied.
236 * \param[in,out] gpu_nbv The nonbonded data GPU structure.
237 * \param[in] locality Copy coordinates for local or non-local atoms.
238 * \param[in] coordinatesHost Host-side coordinates in plain rvec format.
241 void nbnxn_gpu_copy_x_to_gpu(const Nbnxm::Grid gmx_unused &grid,
242 gmx_nbnxn_gpu_t gmx_unused *gpu_nbv,
243 Nbnxm::AtomLocality gmx_unused locality,
244 const rvec gmx_unused *coordinatesHost) CUDA_FUNC_TERM;
246 /*! \brief Getter for the device coordinates buffer.
248 * \todo This will be removed as the management of the buffers is taken out of the NBNXM module.
250 * \param[in] gpu_nbv The nonbonded data GPU structure.
252 * \returns Device coordinates buffer in plain rvec format.
255 DeviceBuffer<float> nbnxn_gpu_get_x_gpu(gmx_nbnxn_gpu_t gmx_unused *gpu_nbv) CUDA_FUNC_TERM_WITH_RETURN(DeviceBuffer<float> {});
258 /*! \brief X buffer operations on GPU: performs conversion from rvec to nb format.
260 * \param[in] grid Grid to be converted.
261 * \param[in] setFillerCoords If the filler coordinates are used.
262 * \param[in,out] gpu_nbv The nonbonded data GPU structure.
263 * \param[in] coordinatesDevice Device-side coordinates in plain rvec format.
264 * \param[in] locality Copy coordinates for local or non-local atoms.
265 * \param[in] gridId Index of the grid being converted.
266 * \param[in] numColumnsMax Maximum number of columns in the grid.
269 void nbnxn_gpu_x_to_nbat_x(const Nbnxm::Grid gmx_unused &grid,
270 bool gmx_unused setFillerCoords,
271 gmx_nbnxn_gpu_t gmx_unused *gpu_nbv,
272 DeviceBuffer<float> gmx_unused coordinatesDevice,
273 Nbnxm::AtomLocality gmx_unused locality,
274 int gmx_unused gridId,
275 int gmx_unused numColumnsMax) CUDA_FUNC_TERM;
277 /*! \brief Sync the nonlocal stream with dependent tasks in the local queue.
278 * \param[in] nb The nonbonded data GPU structure
279 * \param[in] interactionLocality Local or NonLocal sync point
282 void nbnxnInsertNonlocalGpuDependency(const gmx_nbnxn_gpu_t gmx_unused *nb,
283 InteractionLocality gmx_unused interactionLocality) CUDA_FUNC_TERM;
285 /*! \brief Set up internal flags that indicate what type of short-range work there is.
287 * As nonbondeds and bondeds share input/output buffers and GPU queues,
288 * both are considered when checking for work in the current domain.
290 * This function is expected to be called every time the work-distribution
291 * can change (i.e. at search/domain decomposition steps).
293 * \param[inout] nb Pointer to the nonbonded GPU data structure
294 * \param[in] gpuBonded Pointer to the GPU bonded data structure
295 * \param[in] iLocality Interaction locality identifier
298 void setupGpuShortRangeWork(gmx_nbnxn_gpu_t gmx_unused *nb,
299 const gmx::GpuBonded gmx_unused *gpuBonded,
300 Nbnxm::InteractionLocality gmx_unused iLocality) GPU_FUNC_TERM;
302 /*! \brief Returns true if there is GPU short-range work for the given atom locality.
304 * Note that as, unlike nonbonded tasks, bonded tasks are not split into local/nonlocal,
305 * and therefore if there are GPU offloaded bonded interactions, this function will return
306 * true for both local and nonlocal atom range.
308 * \param[inout] nb Pointer to the nonbonded GPU data structure
309 * \param[in] aLocality Atom locality identifier
312 bool haveGpuShortRangeWork(const gmx_nbnxn_gpu_t gmx_unused *nb,
313 Nbnxm::AtomLocality gmx_unused aLocality) GPU_FUNC_TERM_WITH_RETURN(false);
315 /*! \brief Initialization for F buffer operations on GPU */
317 void nbnxn_gpu_init_add_nbat_f_to_f(const int gmx_unused *cell,
318 gmx_nbnxn_gpu_t gmx_unused *gpu_nbv,
319 int gmx_unused natoms_total) CUDA_FUNC_TERM;
321 /*! \brief Force buffer operations on GPU.
323 * Transforms non-bonded forces into plain rvec format and add all the force components to the total
326 * \param[in] atomLocality If the reduction should be performed on local or non-local atoms.
327 * \param[in] totalForcesDevice Device buffer to accumulate resulting force.
328 * \param[in] gpu_nbv The NBNXM GPU data structure.
329 * \param[in] pmeForcesDevice Device buffer with PME forces.
330 * \param[in] pmeForcesReady Event that signals when the PME forces are ready for the reduction.
331 * \param[in] atomStart Index of the first atom to reduce forces for.
332 * \param[in] numAtoms Number of atoms to reduce forces for.
333 * \param[in] useGpuFPmeReduction Whether PME forces should be added.
334 * \param[in] accumulateForce Whether there are usefull data already in the total force buffer.
338 void nbnxn_gpu_add_nbat_f_to_f(AtomLocality gmx_unused atomLocality,
339 DeviceBuffer<float> gmx_unused totalForcesDevice,
340 gmx_nbnxn_gpu_t gmx_unused *gpu_nbv,
341 void gmx_unused *pmeForcesDevice,
342 GpuEventSynchronizer gmx_unused *pmeForcesReady,
343 int gmx_unused atomStart,
344 int gmx_unused numAtoms,
345 bool gmx_unused useGpuFPmeReduction,
346 bool gmx_unused accumulateForce) CUDA_FUNC_TERM;
348 /*! \brief Getter for the device coordinates buffer.
350 * \todo This will be removed as the management of the buffers is taken out of the NBNXM module.
352 * \param[in] gpu_nbv The nonbonded data GPU structure.
354 * \returns Device coordinates buffer in plain rvec format.
357 DeviceBuffer<float> nbnxn_gpu_get_f_gpu(gmx_nbnxn_gpu_t gmx_unused *gpu_nbv) CUDA_FUNC_TERM_WITH_RETURN(DeviceBuffer<float> {});
359 /*! \brief Copy force buffer from CPU to GPU */
361 void nbnxn_launch_copy_f_to_gpu(AtomLocality gmx_unused atomLocality,
362 const Nbnxm::GridSet gmx_unused &gridSet,
363 gmx_nbnxn_gpu_t gmx_unused *nb,
364 rvec gmx_unused *f) CUDA_FUNC_TERM;
366 /*! \brief Copy force buffer from GPU to CPU */
368 void nbnxn_launch_copy_f_from_gpu(AtomLocality gmx_unused atomLocality,
369 const Nbnxm::GridSet gmx_unused &gridSet,
370 gmx_nbnxn_gpu_t gmx_unused *nb,
371 rvec gmx_unused *f) CUDA_FUNC_TERM;
373 /*! \brief Asynchronous launch of copying coordinate buffer from GPU to CPU
374 * \param[in] atomLocality Locality for data trasnfer
375 * \param[in] gridSet The Grid Set data object
376 * \param[in] nb The nonbonded data GPU structure
377 * \param[out] x Coordinate buffer on CPU
380 void nbnxn_launch_copy_x_from_gpu(AtomLocality gmx_unused atomLocality,
381 const Nbnxm::GridSet gmx_unused &gridSet,
382 gmx_nbnxn_gpu_t gmx_unused *nb,
383 rvec gmx_unused *x) CUDA_FUNC_TERM;
385 /*! \brief Wait for GPU stream to complete */
387 void nbnxn_wait_for_gpu_force_reduction(AtomLocality gmx_unused atomLocality,
388 gmx_nbnxn_gpu_t gmx_unused *nb) CUDA_FUNC_TERM;
390 /*! \brief sync CPU thread on coordinate copy to device
391 * \param[in] nb The nonbonded data GPU structure
394 void nbnxn_wait_x_on_device(gmx_nbnxn_gpu_t gmx_unused *nb) CUDA_FUNC_TERM;
396 /*! \brief return pointer to event recorded when coordinates have been copied to device
397 * \param[in] nb The nonbonded data GPU structure
400 void* nbnxn_get_x_on_device_event(const gmx_nbnxn_gpu_t gmx_unused *nb) CUDA_FUNC_TERM_WITH_RETURN(nullptr);
402 /*! \brief return GPU pointer to x in rvec format
403 * \param[in] nb The nonbonded data GPU structure
406 void* nbnxn_get_gpu_xrvec(gmx_nbnxn_gpu_t gmx_unused *nb) CUDA_FUNC_TERM_WITH_RETURN(nullptr);
408 /*! \brief Wait for non-local copy of coordinate buffer from device to host
409 * \param[in] nb The nonbonded data GPU structure
412 void nbnxn_wait_nonlocal_x_copy_D2H_done(gmx_nbnxn_gpu_t gmx_unused *nb) CUDA_FUNC_TERM;