* state with other run parameters and settings. As a result, the initial state
* may be altered or an error may be thrown if incompatibility of options is detected.
*
- * \param [in] mdlog Logger.
- * \param [in] dlbOption Enum value for the DLB option.
- * \param [in] bRecordLoad True if the load balancer is recording load information.
- * \param [in] mdrunOptions Options for mdrun.
- * \param [in] inputrec Pointer mdrun to input parameters.
- * \returns DLB initial/startup state.
+ * \param [in] mdlog Logger.
+ * \param [in] dlbOption Enum value for the DLB option.
+ * \param [in] bRecordLoad True if the load balancer is recording load information.
+ * \param [in] mdrunOptions Options for mdrun.
+ * \param [in] inputrec Pointer mdrun to input parameters.
+ * \param [in] directGpuCommUsedWithGpuUpdate Direct GPU halo exchange and GPU update enabled
+ * \returns DLB initial/startup state.
*/
static DlbState determineInitialDlbState(const gmx::MDLogger& mdlog,
DlbOption dlbOption,
gmx_bool bRecordLoad,
const gmx::MdrunOptions& mdrunOptions,
- const t_inputrec& inputrec)
+ const t_inputrec& inputrec,
+ const bool directGpuCommUsedWithGpuUpdate)
{
DlbState dlbState = DlbState::offCanTurnOn;
default: gmx_incons("Invalid dlbOption enum value");
}
+ // P2P GPU comm + GPU update leads to case in which we enqueue async work for multiple timesteps
+ // DLB needs to be disabled in that case
+ if (directGpuCommUsedWithGpuUpdate)
+ {
+ std::string reasonStr =
+ "it is not supported with GPU direct communication + GPU update enabled.";
+ return forceDlbOffOrBail(dlbState, reasonStr, mdlog);
+ }
+
/* Reruns don't support DLB: bail or override auto mode */
if (mdrunOptions.rerun)
{
static DDSettings getDDSettings(const gmx::MDLogger& mdlog,
const DomdecOptions& options,
const gmx::MdrunOptions& mdrunOptions,
- const t_inputrec& ir)
+ const t_inputrec& ir,
+ const bool directGpuCommUsedWithGpuUpdate)
{
DDSettings ddSettings;
ddSettings.recordLoad = (wallcycle_have_counter() && recload > 0);
}
- ddSettings.initialDlbState =
- determineInitialDlbState(mdlog, options.dlbOption, ddSettings.recordLoad, mdrunOptions, ir);
+ ddSettings.initialDlbState = determineInitialDlbState(
+ mdlog, options.dlbOption, ddSettings.recordLoad, mdrunOptions, ir, directGpuCommUsedWithGpuUpdate);
GMX_LOG(mdlog.info)
.appendTextFormatted("Dynamic load balancing: %s",
enumValueToString(ddSettings.initialDlbState));
real maxUpdateGroupRadius,
ArrayRef<const RVec> xGlobal,
bool useGpuForNonbonded,
- bool useGpuForPme);
+ bool useGpuForPme,
+ bool directGpuCommUsedWithGpuUpdate);
//! Build the resulting DD manager
gmx_domdec_t* build(LocalAtomSetManager* atomSets);
const real maxUpdateGroupRadius,
ArrayRef<const RVec> xGlobal,
bool useGpuForNonbonded,
- bool useGpuForPme) :
+ bool useGpuForPme,
+ bool directGpuCommUsedWithGpuUpdate) :
mdlog_(mdlog), cr_(cr), options_(options), mtop_(mtop), ir_(ir), notifiers_(notifiers)
{
GMX_LOG(mdlog_.info).appendTextFormatted("\nInitializing Domain Decomposition on %d ranks", cr_->sizeOfDefaultCommunicator);
- ddSettings_ = getDDSettings(mdlog_, options_, mdrunOptions, ir_);
+ ddSettings_ = getDDSettings(mdlog_, options_, mdrunOptions, ir_, directGpuCommUsedWithGpuUpdate);
if (ddSettings_.eFlop > 1)
{
const real maxUpdateGroupRadius,
ArrayRef<const RVec> xGlobal,
const bool useGpuForNonbonded,
- const bool useGpuForPme) :
+ const bool useGpuForPme,
+ const bool directGpuCommUsedWithGpuUpdate) :
impl_(new Impl(mdlog,
cr,
options,
maxUpdateGroupRadius,
xGlobal,
useGpuForNonbonded,
- useGpuForPme))
+ useGpuForPme,
+ directGpuCommUsedWithGpuUpdate))
{
}
systemHasConstraintsOrVsites(mtop),
cutoffMargin);
+ try
+ {
+ const bool haveFrozenAtoms = inputrecFrozenAtoms(inputrec.get());
+
+ useGpuForUpdate = decideWhetherToUseGpuForUpdate(useDomainDecomposition,
+ updateGroups.useUpdateGroups(),
+ pmeRunMode,
+ domdecOptions.numPmeRanks > 0,
+ useGpuForNonbonded,
+ updateTarget,
+ gpusWereDetected,
+ *inputrec,
+ mtop,
+ doEssentialDynamics,
+ gmx_mtop_ftype_count(mtop, F_ORIRES) > 0,
+ haveFrozenAtoms,
+ doRerun,
+ devFlags,
+ mdlog);
+ }
+ GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
+
+ bool useGpuDirectHalo = false;
+
+ if (useGpuForNonbonded)
+ {
+ // cr->npmenodes is not yet initialized.
+ // domdecOptions.numPmeRanks == -1 results in 0 separate PME ranks when useGpuForNonbonded is true.
+ // Todo: remove this assumption later once auto mode has support for separate PME rank
+ const int numPmeRanks = domdecOptions.numPmeRanks > 0 ? domdecOptions.numPmeRanks : 0;
+ bool havePPDomainDecomposition = (cr->sizeOfDefaultCommunicator - numPmeRanks) > 1;
+ useGpuDirectHalo = decideWhetherToUseGpuForHalo(devFlags,
+ havePPDomainDecomposition,
+ useGpuForNonbonded,
+ useModularSimulator,
+ doRerun,
+ EI_ENERGY_MINIMIZATION(inputrec->eI));
+ }
+
// This builder is necessary while we have multi-part construction
// of DD. Before DD is constructed, we use the existence of
// the builder object to indicate that further construction of DD
std::unique_ptr<DomainDecompositionBuilder> ddBuilder;
if (useDomainDecomposition)
{
- ddBuilder = std::make_unique<DomainDecompositionBuilder>(
+ // P2P GPU comm + GPU update leads to case in which we enqueue async work for multiple
+ // timesteps. DLB needs to be disabled in that case
+ const bool directGpuCommUsedWithGpuUpdate = GMX_THREAD_MPI && useGpuDirectHalo && useGpuForUpdate;
+ ddBuilder = std::make_unique<DomainDecompositionBuilder>(
mdlog,
cr,
domdecOptions,
updateGroups.maxUpdateGroupRadius(),
positionsFromStatePointer(globalState.get()),
useGpuForNonbonded,
- useGpuForPme);
+ useGpuForPme,
+ directGpuCommUsedWithGpuUpdate);
}
else
{
}
}
- // The GPU update is decided here because we need to know whether the constraints or
- // SETTLEs can span across the domain borders (i.e. whether or not update groups are
- // defined). This is only known after DD is initialized, hence decision on using GPU
- // update is done so late.
- try
- {
- const bool haveFrozenAtoms = inputrecFrozenAtoms(inputrec.get());
-
- useGpuForUpdate = decideWhetherToUseGpuForUpdate(useDomainDecomposition,
- updateGroups.useUpdateGroups(),
- pmeRunMode,
- domdecOptions.numPmeRanks > 0,
- useGpuForNonbonded,
- updateTarget,
- gpusWereDetected,
- *inputrec,
- mtop,
- doEssentialDynamics,
- gmx_mtop_ftype_count(mtop, F_ORIRES) > 0,
- haveFrozenAtoms,
- doRerun,
- devFlags,
- mdlog);
- }
- GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
-
const bool printHostName = (cr->nnodes > 1);
gpuTaskAssignments.reportGpuUsage(mdlog, printHostName, useGpuForBonded, pmeRunMode, useGpuForUpdate);
MdrunScheduleWorkload runScheduleWork;
- bool useGpuDirectHalo = decideWhetherToUseGpuForHalo(devFlags,
- havePPDomainDecomposition(cr),
- useGpuForNonbonded,
- useModularSimulator,
- doRerun,
- EI_ENERGY_MINIMIZATION(inputrec->eI));
-
// Also populates the simulation constant workload description.
// Note: currently the default duty is DUTY_PP | DUTY_PME for all simulations, including those without PME,
// so this boolean is sufficient on all ranks to determine whether separate PME ranks are used,
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff