2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2017,2018,2019,2020,2021, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
36 * \brief Declares functionality for deciding whether tasks will run on GPUs.
38 * \author Mark Abraham <mark.j.abraham@gmail.com>
39 * \ingroup module_taskassignment
43 #ifndef GMX_TASKASSIGNMENT_DECIDEGPUUSAGE_H
44 #define GMX_TASKASSIGNMENT_DECIDEGPUUSAGE_H
51 enum class PmeRunMode;
58 //! Record where a compute task is targetted.
59 enum class TaskTarget : int
66 //! Help pass GPU-emulation parameters with type safety.
67 enum class EmulateGpuNonbonded : bool
69 //! Do not emulate GPUs.
76 * \brief Structure that holds boolean flags corresponding to the development
77 * features present enabled through environment variables.
80 struct DevelopmentFeatureFlags
82 //! True if the Buffer ops development feature is enabled
83 // TODO: when the trigger of the buffer ops offload is fully automated this should go away
84 bool enableGpuBufferOps = false;
85 //! If true, forces 'mdrun -update auto' default to 'gpu'
86 bool forceGpuUpdateDefault = false;
87 //! True if the GPU halo exchange development feature is enabled
88 bool enableGpuHaloExchange = false;
89 //! True if the PME PP direct communication GPU development feature is enabled
90 bool enableGpuPmePPComm = false;
91 //! True if the CUDA-aware MPI is being used for GPU direct communication feature
92 bool usingCudaAwareMpi = false;
98 /*! \brief Decide whether this thread-MPI simulation will run
99 * nonbonded tasks on GPUs.
101 * The number of GPU tasks and devices influences both the choice of
102 * the number of ranks, and checks upon any such choice made by the
103 * user. So we need to consider this before any automated choice of
104 * the number of thread-MPI ranks.
106 * \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign
107 * short-ranged nonbonded interaction tasks.
108 * \param[in] haveAvailableDevices Whether there are available devices.
109 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
110 * \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded
112 * \param[in] buildSupportsNonbondedOnGpu Whether GROMACS was built with GPU support.
113 * \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is
114 * useful for this calculation.
115 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
117 * \returns Whether the simulation will run nonbonded tasks on GPUs.
119 * \throws std::bad_alloc If out of memory
120 * InconsistentInputError If the user requirements are inconsistent. */
121 bool decideWhetherToUseGpusForNonbondedWithThreadMpi(TaskTarget nonbondedTarget,
122 bool haveAvailableDevices,
123 const std::vector<int>& userGpuTaskAssignment,
124 EmulateGpuNonbonded emulateGpuNonbonded,
125 bool buildSupportsNonbondedOnGpu,
126 bool nonbondedOnGpuIsUseful,
127 int numRanksPerSimulation);
129 /*! \brief Decide whether this thread-MPI simulation will run
132 * The number of GPU tasks and devices influences both the choice of
133 * the number of ranks, and checks upon any such choice made by the
134 * user. So we need to consider this before any automated choice of
135 * the number of thread-MPI ranks.
137 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
138 * \param[in] pmeTarget The user's choice for mdrun -pme for where to assign
139 * long-ranged PME nonbonded interaction tasks.
140 * \param[in] pmeFftTarget The user's choice for mdrun -pmefft for where to run FFT.
141 * \param[in] numDevicesToUse The number of compatible GPUs that the user permitted us to use.
142 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
143 * \param[in] hardwareInfo Hardware information
144 * \param[in] inputrec The user input
145 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
146 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
148 * \returns Whether the simulation will run PME tasks on GPUs.
150 * \throws std::bad_alloc If out of memory
151 * InconsistentInputError If the user requirements are inconsistent. */
152 bool decideWhetherToUseGpusForPmeWithThreadMpi(bool useGpuForNonbonded,
153 TaskTarget pmeTarget,
154 TaskTarget pmeFftTarget,
156 const std::vector<int>& userGpuTaskAssignment,
157 const gmx_hw_info_t& hardwareInfo,
158 const t_inputrec& inputrec,
159 int numRanksPerSimulation,
160 int numPmeRanksPerSimulation);
162 /*! \brief Decide whether the simulation will try to run nonbonded
165 * The final decision cannot be made until after the duty of the rank
166 * is known. But we need to know if nonbonded will run on GPUs for
167 * setting up DD (particularly rlist) and determining duty. If the
168 * user requires GPUs for the tasks of that duty, then it will be an
169 * error when none are found.
171 * With thread-MPI, calls have been made to
172 * decideWhetherToUseGpusForNonbondedWithThreadMpi() and
173 * decideWhetherToUseGpusForPmeWithThreadMpi() to help determine
174 * the number of ranks and run some checks, but the final
175 * decision is made in this routine, along with many more
176 * consistency checks.
178 * \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign short-ranged nonbonded interaction tasks.
179 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
180 * \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded interactions.
181 * \param[in] buildSupportsNonbondedOnGpu Whether GROMACS was build with GPU support.
182 * \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is useful for this calculation.
183 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
185 * \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
187 * \throws std::bad_alloc If out of memory
188 * InconsistentInputError If the user requirements are inconsistent. */
189 bool decideWhetherToUseGpusForNonbonded(TaskTarget nonbondedTarget,
190 const std::vector<int>& userGpuTaskAssignment,
191 EmulateGpuNonbonded emulateGpuNonbonded,
192 bool buildSupportsNonbondedOnGpu,
193 bool nonbondedOnGpuIsUseful,
194 bool gpusWereDetected);
196 /*! \brief Decide whether the simulation will try to run tasks of
197 * different types on GPUs.
199 * The final decision cannot be made until after the duty of the rank
200 * is known. But we need to know if nonbonded will run on GPUs for
201 * setting up DD (particularly rlist) and determining duty. If the
202 * user requires GPUs for the tasks of that duty, then it will be an
203 * error when none are found.
205 * With thread-MPI, calls have been made to
206 * decideWhetherToUseGpusForNonbondedWithThreadMpi() and
207 * decideWhetherToUseGpusForPmeWithThreadMpi() to help determine
208 * the number of ranks and run some checks, but the final
209 * decision is made in this routine, along with many more
210 * consistency checks.
212 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
213 * \param[in] pmeTarget The user's choice for mdrun -pme for where to assign long-ranged PME nonbonded interaction tasks.
214 * \param[in] pmeFftTarget The user's choice for mdrun -pmefft for where to do FFT for PME.
215 * \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
216 * \param[in] hardwareInfo Hardware information
217 * \param[in] inputrec The user input
218 * \param[in] numRanksPerSimulation The number of ranks in each simulation.
219 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
220 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
222 * \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
224 * \throws std::bad_alloc If out of memory
225 * InconsistentInputError If the user requirements are inconsistent. */
226 bool decideWhetherToUseGpusForPme(bool useGpuForNonbonded,
227 TaskTarget pmeTarget,
228 TaskTarget pmeFftTarget,
229 const std::vector<int>& userGpuTaskAssignment,
230 const gmx_hw_info_t& hardwareInfo,
231 const t_inputrec& inputrec,
232 int numRanksPerSimulation,
233 int numPmeRanksPerSimulation,
234 bool gpusWereDetected);
236 /*! \brief Determine PME run mode.
238 * Given the PME task assignment in \p useGpuForPme and the user-provided
239 * FFT task target in \p pmeFftTarget, returns a PME run mode for the
240 * current run. It also checks the compatibility of the two.
242 * \note Aborts the run upon incompatible values of \p useGpuForPme and \p pmeFftTarget.
244 * \param[in] useGpuForPme PME task assignment, true if PME task is mapped to the GPU.
245 * \param[in] pmeFftTarget The user's choice for -pmefft for where to assign the FFT
246 * work of the PME task.
247 * \param[in] inputrec The user input record
249 PmeRunMode determinePmeRunMode(bool useGpuForPme, const TaskTarget& pmeFftTarget, const t_inputrec& inputrec);
251 /*! \brief Decide whether the simulation will try to run bonded tasks on GPUs.
253 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
254 * \param[in] useGpuForPme Whether GPUs will be used for PME interactions.
255 * \param[in] bondedTarget The user's choice for mdrun -bonded for where to assign tasks.
256 * \param[in] inputrec The user input.
257 * \param[in] mtop The global topology.
258 * \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation, can be -1 for auto.
259 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
261 * \returns Whether the simulation will run bondeded tasks on GPUs.
263 * \throws std::bad_alloc If out of memory
264 * InconsistentInputError If the user requirements are inconsistent. */
265 bool decideWhetherToUseGpusForBonded(bool useGpuForNonbonded,
267 TaskTarget bondedTarget,
268 const t_inputrec& inputrec,
269 const gmx_mtop_t& mtop,
270 int numPmeRanksPerSimulation,
271 bool gpusWereDetected);
273 /*! \brief Decide whether to use GPU for update.
275 * \param[in] isDomainDecomposition Whether there more than one domain.
276 * \param[in] useUpdateGroups If the constraints can be split across domains.
277 * \param[in] pmeRunMode PME running mode: CPU, GPU or mixed.
278 * \param[in] havePmeOnlyRank If there is a PME-only rank in the simulation.
279 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
280 * \param[in] updateTarget User choice for running simulation on GPU.
281 * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
282 * \param[in] inputrec The user input.
283 * \param[in] mtop The global topology.
284 * \param[in] useEssentialDynamics If essential dynamics is active.
285 * \param[in] doOrientationRestraints If orientation restraints are enabled.
286 * \param[in] haveFrozenAtoms If this simulation has frozen atoms (see Issue #3920).
287 * \param[in] doRerun It this is a rerun.
288 * \param[in] devFlags GPU development / experimental feature flags.
289 * \param[in] mdlog MD logger.
291 * \returns Whether complete simulation can be run on GPU.
292 * \throws std::bad_alloc If out of memory
293 * InconsistentInputError If the user requirements are inconsistent.
295 bool decideWhetherToUseGpuForUpdate(bool isDomainDecomposition,
296 bool useUpdateGroups,
297 PmeRunMode pmeRunMode,
298 bool havePmeOnlyRank,
299 bool useGpuForNonbonded,
300 TaskTarget updateTarget,
301 bool gpusWereDetected,
302 const t_inputrec& inputrec,
303 const gmx_mtop_t& mtop,
304 bool useEssentialDynamics,
305 bool doOrientationRestraints,
306 bool haveFrozenAtoms,
308 const DevelopmentFeatureFlags& devFlags,
309 const gmx::MDLogger& mdlog);
312 /*! \brief Decide whether to use GPU for halo exchange.
314 * \param[in] devFlags GPU development / experimental feature flags.
315 * \param[in] havePPDomainDecomposition Whether PP domain decomposition is in use.
316 * \param[in] useGpuForNonbonded Whether GPUs will be used for nonbonded interactions.
317 * \param[in] useModularSimulator Whether modularsimulator is in use.
318 * \param[in] doRerun Whether this is a rerun.
319 * \param[in] haveEnergyMinimization Whether energy minimization is in use.
321 * \returns Whether halo exchange can be run on GPU.
323 bool decideWhetherToUseGpuForHalo(const DevelopmentFeatureFlags& devFlags,
324 bool havePPDomainDecomposition,
325 bool useGpuForNonbonded,
326 bool useModularSimulator,
328 bool haveEnergyMinimization);