2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
38 #include "cudautils.cuh"
43 #include "gromacs/gpu_utils/cuda_arch_utils.cuh"
44 #include "gromacs/gpu_utils/gpu_utils.h"
45 #include "gromacs/utility/gmxassert.h"
47 /*** Generic CUDA data operation wrappers ***/
49 // TODO: template on transferKind to avoid runtime conditionals
50 int cu_copy_D2H(void* h_dest, void* d_src, size_t bytes, GpuApiCallBehavior transferKind, cudaStream_t s = nullptr)
54 if (h_dest == nullptr || d_src == nullptr || bytes == 0)
61 case GpuApiCallBehavior::Async:
62 GMX_ASSERT(isHostMemoryPinned(h_dest), "Destination buffer was not pinned for CUDA");
63 stat = cudaMemcpyAsync(h_dest, d_src, bytes, cudaMemcpyDeviceToHost, s);
64 CU_RET_ERR(stat, "DtoH cudaMemcpyAsync failed");
67 case GpuApiCallBehavior::Sync:
68 stat = cudaMemcpy(h_dest, d_src, bytes, cudaMemcpyDeviceToHost);
69 CU_RET_ERR(stat, "DtoH cudaMemcpy failed");
78 int cu_copy_D2H_sync(void* h_dest, void* d_src, size_t bytes)
80 return cu_copy_D2H(h_dest, d_src, bytes, GpuApiCallBehavior::Sync);
84 * The copy is launched in stream s or if not specified, in stream 0.
86 int cu_copy_D2H_async(void* h_dest, void* d_src, size_t bytes, cudaStream_t s = nullptr)
88 return cu_copy_D2H(h_dest, d_src, bytes, GpuApiCallBehavior::Async, s);
91 // TODO: template on transferKind to avoid runtime conditionals
92 int cu_copy_H2D(void* d_dest, const void* h_src, size_t bytes, GpuApiCallBehavior transferKind, cudaStream_t s = nullptr)
96 if (d_dest == nullptr || h_src == nullptr || bytes == 0)
101 switch (transferKind)
103 case GpuApiCallBehavior::Async:
104 GMX_ASSERT(isHostMemoryPinned(h_src), "Source buffer was not pinned for CUDA");
105 stat = cudaMemcpyAsync(d_dest, h_src, bytes, cudaMemcpyHostToDevice, s);
106 CU_RET_ERR(stat, "HtoD cudaMemcpyAsync failed");
109 case GpuApiCallBehavior::Sync:
110 stat = cudaMemcpy(d_dest, h_src, bytes, cudaMemcpyHostToDevice);
111 CU_RET_ERR(stat, "HtoD cudaMemcpy failed");
120 int cu_copy_H2D_sync(void* d_dest, const void* h_src, size_t bytes)
122 return cu_copy_H2D(d_dest, h_src, bytes, GpuApiCallBehavior::Sync);
126 * The copy is launched in stream s or if not specified, in stream 0.
128 int cu_copy_H2D_async(void* d_dest, const void* h_src, size_t bytes, cudaStream_t s = nullptr)
130 return cu_copy_H2D(d_dest, h_src, bytes, GpuApiCallBehavior::Async, s);
133 /*! \brief Set up texture object for an array of type T.
135 * Set up texture object for an array of type T and bind it to the device memory
136 * \p d_ptr points to.
138 * \tparam[in] T Raw data type
139 * \param[out] texObj texture object to initialize
140 * \param[in] d_ptr pointer to device global memory to bind \p texObj to
141 * \param[in] sizeInBytes size of memory area to bind \p texObj to
144 static void setup1DTexture(cudaTextureObject_t& texObj, void* d_ptr, size_t sizeInBytes)
146 assert(!c_disableCudaTextures);
152 memset(&rd, 0, sizeof(rd));
153 rd.resType = cudaResourceTypeLinear;
154 rd.res.linear.devPtr = d_ptr;
155 rd.res.linear.desc = cudaCreateChannelDesc<T>();
156 rd.res.linear.sizeInBytes = sizeInBytes;
158 memset(&td, 0, sizeof(td));
159 td.readMode = cudaReadModeElementType;
160 stat = cudaCreateTextureObject(&texObj, &rd, &td, nullptr);
161 CU_RET_ERR(stat, "cudaCreateTextureObject failed");
165 void initParamLookupTable(T*& d_ptr, cudaTextureObject_t& texObj, const T* h_ptr, int numElem)
167 const size_t sizeInBytes = numElem * sizeof(*d_ptr);
168 cudaError_t stat = cudaMalloc((void**)&d_ptr, sizeInBytes);
169 CU_RET_ERR(stat, "cudaMalloc failed in initParamLookupTable");
170 cu_copy_H2D_sync(d_ptr, (void*)h_ptr, sizeInBytes);
172 if (!c_disableCudaTextures)
174 setup1DTexture<T>(texObj, d_ptr, sizeInBytes);
179 void destroyParamLookupTable(T* d_ptr, cudaTextureObject_t texObj)
181 if (!c_disableCudaTextures)
183 CU_RET_ERR(cudaDestroyTextureObject(texObj), "cudaDestroyTextureObject on texObj failed");
185 CU_RET_ERR(cudaFree(d_ptr), "cudaFree failed");
188 /*! \brief Add explicit instantiations of init/destroyParamLookupTable() here as needed.
189 * One should also verify that the result of cudaCreateChannelDesc<T>() during texture setup
190 * looks reasonable, when instantiating the templates for new types - just in case.
192 template void initParamLookupTable<float>(float*&, cudaTextureObject_t&, const float*, int);
193 template void destroyParamLookupTable<float>(float*, cudaTextureObject_t);
194 template void initParamLookupTable<int>(int*&, cudaTextureObject_t&, const int*, int);
195 template void destroyParamLookupTable<int>(int*, cudaTextureObject_t);