The logic was flawed such that GPUs were "selected" for use even
though none had been detected. That led to the GPU behaviour of
avoiding using separate PME ranks.
Also made a minor fix to the logic for emulation. The new
interpretation of mdrun -gpu_id does not need to trigger an error when
GPU IDs have been supplied along with the emulation environmnet
variable.
Fixes #2315
Change-Id: I68da27c9bfef9f73b9dae4f04f196066d2efb1e2
}
bool decideWhetherToUseGpusForNonbonded(const TaskTarget nonbondedTarget,
}
bool decideWhetherToUseGpusForNonbonded(const TaskTarget nonbondedTarget,
- const std::vector<int> &gpuIdsToUse,
const std::vector<int> &userGpuTaskAssignment,
const EmulateGpuNonbonded emulateGpuNonbonded,
const bool usingVerletScheme,
const std::vector<int> &userGpuTaskAssignment,
const EmulateGpuNonbonded emulateGpuNonbonded,
const bool usingVerletScheme,
- const bool nonbondedOnGpuIsUseful)
+ const bool nonbondedOnGpuIsUseful,
+ const bool gpusWereDetected)
{
if (nonbondedTarget == TaskTarget::Cpu)
{
{
if (nonbondedTarget == TaskTarget::Cpu)
{
("Nonbonded interactions on the GPU were required, which is inconsistent "
"with choosing emulation. Make no more than one of these choices."));
}
("Nonbonded interactions on the GPU were required, which is inconsistent "
"with choosing emulation. Make no more than one of these choices."));
}
- if (!gpuIdsToUse.empty() || !userGpuTaskAssignment.empty())
+ if (!userGpuTaskAssignment.empty())
{
GMX_THROW(InconsistentInputError
("GPU ID usage was specified, as was GPU emulation. Make no more than one of these choices."));
{
GMX_THROW(InconsistentInputError
("GPU ID usage was specified, as was GPU emulation. Make no more than one of these choices."));
- // We still don't know whether it is an error if no GPUs are found
- // because we don't know the duty of this rank, yet. For example,
- // a node with only PME ranks and -pme cpu is OK if there are not
- // GPUs.
+ if (nonbondedTarget == TaskTarget::Gpu)
+ {
+ // We still don't know whether it is an error if no GPUs are found
+ // because we don't know the duty of this rank, yet. For example,
+ // a node with only PME ranks and -pme cpu is OK if there are not
+ // GPUs.
+ return true;
+ }
- // If we get here, then the user permitted or required GPUs.
- return true;
+ // If we get here, then the user permitted GPUs, which we should
+ // use for nonbonded interactions.
+ return gpusWereDetected;
}
bool decideWhetherToUseGpusForPme(const bool useGpuForNonbonded,
}
bool decideWhetherToUseGpusForPme(const bool useGpuForNonbonded,
const std::vector<int> &userGpuTaskAssignment,
const bool canUseGpuForPme,
const int numRanksPerSimulation,
const std::vector<int> &userGpuTaskAssignment,
const bool canUseGpuForPme,
const int numRanksPerSimulation,
- const int numPmeRanksPerSimulation)
+ const int numPmeRanksPerSimulation,
+ const bool gpusWereDetected)
{
if (pmeTarget == TaskTarget::Cpu)
{
{
if (pmeTarget == TaskTarget::Cpu)
{
+ // If we get here, then the user permitted GPUs.
if (numRanksPerSimulation == 1)
{
if (numRanksPerSimulation == 1)
{
- // PME can run well on a single GPU shared with NB when
- // there is one rank, so we permit mdrun to try that.
- return true;
+ // PME can run well on a single GPU shared with NB when there
+ // is one rank, so we permit mdrun to try that if we have
+ // detected GPUs.
+ return gpusWereDetected;
}
// Not enough support for PME on GPUs for anything else
}
// Not enough support for PME on GPUs for anything else
* consistency checks.
*
* \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign short-ranged nonbonded interaction tasks.
* consistency checks.
*
* \param[in] nonbondedTarget The user's choice for mdrun -nb for where to assign short-ranged nonbonded interaction tasks.
- * \param[in] gpuIdsToUse The compatible GPUs that the user permitted us to use.
* \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
* \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded interactions.
* \param[in] usingVerletScheme Whether the nonbondeds are using the Verlet scheme.
* \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is useful for this calculation.
* \param[in] userGpuTaskAssignment The user-specified assignment of GPU tasks to device IDs.
* \param[in] emulateGpuNonbonded Whether we will emulate GPU calculation of nonbonded interactions.
* \param[in] usingVerletScheme Whether the nonbondeds are using the Verlet scheme.
* \param[in] nonbondedOnGpuIsUseful Whether computing nonbonded interactions on a GPU is useful for this calculation.
+ * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
*
* \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
*
* \throws std::bad_alloc If out of memory
* InconsistentInputError If the user requirements are inconsistent. */
bool decideWhetherToUseGpusForNonbonded(const TaskTarget nonbondedTarget,
*
* \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
*
* \throws std::bad_alloc If out of memory
* InconsistentInputError If the user requirements are inconsistent. */
bool decideWhetherToUseGpusForNonbonded(const TaskTarget nonbondedTarget,
- const std::vector<int> &gpuIdsToUse,
const std::vector<int> &userGpuTaskAssignment,
const EmulateGpuNonbonded emulateGpuNonbonded,
const bool usingVerletScheme,
const std::vector<int> &userGpuTaskAssignment,
const EmulateGpuNonbonded emulateGpuNonbonded,
const bool usingVerletScheme,
- const bool nonbondedOnGpuIsUseful);
+ const bool nonbondedOnGpuIsUseful,
+ const bool gpusWereDetected);
/*! \brief Decide whether the simulation will try to run tasks of
* different types on GPUs.
/*! \brief Decide whether the simulation will try to run tasks of
* different types on GPUs.
* \param[in] canUseGpuForPme Whether the form of PME chosen can run on a GPU
* \param[in] numRanksPerSimulation The number of ranks in each simulation.
* \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
* \param[in] canUseGpuForPme Whether the form of PME chosen can run on a GPU
* \param[in] numRanksPerSimulation The number of ranks in each simulation.
* \param[in] numPmeRanksPerSimulation The number of PME ranks in each simulation.
+ * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node.
*
* \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
*
*
* \returns Whether the simulation will run nonbonded and PME tasks, respectively, on GPUs.
*
const std::vector<int> &userGpuTaskAssignment,
const bool canUseGpuForPme,
const int numRanksPerSimulation,
const std::vector<int> &userGpuTaskAssignment,
const bool canUseGpuForPme,
const int numRanksPerSimulation,
- const int numPmeRanksPerSimulation);
+ const int numPmeRanksPerSimulation,
+ const bool gpusWereDetected);
bool useGpuForPme = false;
try
{
bool useGpuForPme = false;
try
{
- useGpuForNonbonded = decideWhetherToUseGpusForNonbonded(nonbondedTarget, gpuIdsToUse, userGpuTaskAssignment,
+ // It's possible that there are different numbers of GPUs on
+ // different nodes, which is the user's responsibilty to
+ // handle. If unsuitable, we will notice that during task
+ // assignment.
+ bool gpusWereDetected = hwinfo->ngpu_compatible_tot > 0;
+ useGpuForNonbonded = decideWhetherToUseGpusForNonbonded(nonbondedTarget, userGpuTaskAssignment,
emulateGpuNonbonded, inputrec->cutoff_scheme == ecutsVERLET,
emulateGpuNonbonded, inputrec->cutoff_scheme == ecutsVERLET,
- gpuAccelerationOfNonbondedIsUseful(mdlog, inputrec, doRerun));
+ gpuAccelerationOfNonbondedIsUseful(mdlog, inputrec, doRerun),
+ gpusWereDetected);
auto inputSystemHasPme = EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype);
auto canUseGpuForPme = inputSystemHasPme && pme_gpu_supports_input(inputrec, nullptr);
auto inputSystemHasPme = EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype);
auto canUseGpuForPme = inputSystemHasPme && pme_gpu_supports_input(inputrec, nullptr);
- useGpuForPme = decideWhetherToUseGpusForPme(useGpuForNonbonded, pmeTarget, userGpuTaskAssignment, canUseGpuForPme, cr->nnodes, domdecOptions.numPmeRanks);
+ useGpuForPme = decideWhetherToUseGpusForPme(useGpuForNonbonded, pmeTarget, userGpuTaskAssignment,
+ canUseGpuForPme, cr->nnodes, domdecOptions.numPmeRanks,
+ gpusWereDetected);
pmeRunMode = (useGpuForPme ? PmeRunMode::GPU : PmeRunMode::CPU);
if ((pmeRunMode == PmeRunMode::GPU) && (pmeFftTarget == TaskTarget::Cpu))
{
pmeRunMode = (useGpuForPme ? PmeRunMode::GPU : PmeRunMode::CPU);
if ((pmeRunMode == PmeRunMode::GPU) && (pmeFftTarget == TaskTarget::Cpu))
{