Update groups should not be managed from within domdec module, so
shift the responsiblity to runner temporarily to simplify future
changes.
This made clear that some internal fields of domdec module were
inaccurately named. Update groups preclude the possibility of
constraints or vsites being split across domains, but when update
groups are not in use such splits are merely possible, not
assured. The code doesn't change, because we always have to account
for the possiblity that some are split.
Refs #4004
12 files changed:
-bool ddHaveSplitConstraints(const gmx_domdec_t& dd)
+bool ddMayHaveSplitConstraints(const gmx_domdec_t& dd)
- return dd.comm->systemInfo.haveSplitConstraints;
+ return dd.comm->systemInfo.mayHaveSplitConstraints;
}
bool ddUsesUpdateGroups(const gmx_domdec_t& dd)
}
bool ddUsesUpdateGroups(const gmx_domdec_t& dd)
if (systemInfo.useUpdateGroups)
{
if (systemInfo.useUpdateGroups)
{
- systemInfo.haveSplitConstraints = false;
- systemInfo.haveSplitSettles = false;
+ systemInfo.mayHaveSplitConstraints = false;
+ systemInfo.mayHaveSplitSettles = false;
- systemInfo.haveSplitConstraints = (gmx_mtop_ftype_count(mtop, F_CONSTR) > 0
- || gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0);
- systemInfo.haveSplitSettles = (gmx_mtop_ftype_count(mtop, F_SETTLE) > 0);
+ systemInfo.mayHaveSplitConstraints = (gmx_mtop_ftype_count(mtop, F_CONSTR) > 0
+ || gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0);
+ systemInfo.mayHaveSplitSettles = (gmx_mtop_ftype_count(mtop, F_SETTLE) > 0);
}
systemInfo.constraintCommunicationRange = 0;
}
systemInfo.constraintCommunicationRange = 0;
- if (systemInfo.haveSplitConstraints && options.constraintCommunicationRange <= 0)
+ if (systemInfo.mayHaveSplitConstraints && options.constraintCommunicationRange <= 0)
{
/* There is a cell size limit due to the constraints (P-LINCS) */
systemInfo.constraintCommunicationRange = gmx::constr_r_max(mdlog, &mtop, &ir);
{
/* There is a cell size limit due to the constraints (P-LINCS) */
systemInfo.constraintCommunicationRange = gmx::constr_r_max(mdlog, &mtop, &ir);
{
if (options.numCells[XX] <= 0 && (ddGridSetup.numDomains[XX] == 0))
{
{
if (options.numCells[XX] <= 0 && (ddGridSetup.numDomains[XX] == 0))
{
- const bool bC = (systemInfo.haveSplitConstraints
+ const bool bC = (systemInfo.mayHaveSplitConstraints
&& systemInfo.constraintCommunicationRange > systemInfo.minCutoffForMultiBody);
std::string message =
gmx::formatString("Change the number of ranks or mdrun option %s%s%s",
&& systemInfo.constraintCommunicationRange > systemInfo.minCutoffForMultiBody);
std::string message =
gmx::formatString("Change the number of ranks or mdrun option %s%s%s",
(countInterUpdategroupVsites(mtop, comm->systemInfo.updateGroupingsPerMoleculeType) != 0);
if (comm->systemInfo.haveInterDomainBondeds || haveInterDomainVsites
(countInterUpdategroupVsites(mtop, comm->systemInfo.updateGroupingsPerMoleculeType) != 0);
if (comm->systemInfo.haveInterDomainBondeds || haveInterDomainVsites
- || comm->systemInfo.haveSplitConstraints || comm->systemInfo.haveSplitSettles)
+ || comm->systemInfo.mayHaveSplitConstraints || comm->systemInfo.mayHaveSplitSettles)
{
std::string decompUnits;
if (comm->systemInfo.useUpdateGroups)
{
std::string decompUnits;
if (comm->systemInfo.useUpdateGroups)
{
log->writeLineFormatted("%40s %-7s %6.3f nm", "virtual site constructions", "(-rcon)", limit);
}
{
log->writeLineFormatted("%40s %-7s %6.3f nm", "virtual site constructions", "(-rcon)", limit);
}
- if (comm->systemInfo.haveSplitConstraints || comm->systemInfo.haveSplitSettles)
+ if (comm->systemInfo.mayHaveSplitConstraints || comm->systemInfo.mayHaveSplitSettles)
{
std::string separation =
gmx::formatString("atoms separated by up to %d constraints", 1 + ir.nProjOrder);
{
std::string separation =
gmx::formatString("atoms separated by up to %d constraints", 1 + ir.nProjOrder);
/*! \brief Returns the maximum shift for coordinate communication in PME, dim y */
int dd_pme_maxshift_y(const gmx_domdec_t& dd);
/*! \brief Returns the maximum shift for coordinate communication in PME, dim y */
int dd_pme_maxshift_y(const gmx_domdec_t& dd);
-/*! \brief Return whether constraints, not including settles, cross domain boundaries */
-bool ddHaveSplitConstraints(const gmx_domdec_t& dd);
+/*! \brief Return whether constraints, not including settles, may cross domain boundaries */
+bool ddMayHaveSplitConstraints(const gmx_domdec_t& dd);
/*! \brief Return whether update groups are used */
bool ddUsesUpdateGroups(const gmx_domdec_t& dd);
/*! \brief Return whether update groups are used */
bool ddUsesUpdateGroups(const gmx_domdec_t& dd);
// This code should not be called unless this condition is true,
// because that's the only time init_domdec_constraints is
// called...
// This code should not be called unless this condition is true,
// because that's the only time init_domdec_constraints is
// called...
- GMX_RELEASE_ASSERT(dd->comm->systemInfo.haveSplitConstraints || dd->comm->systemInfo.haveSplitSettles,
+ GMX_RELEASE_ASSERT(dd->comm->systemInfo.mayHaveSplitConstraints || dd->comm->systemInfo.mayHaveSplitSettles,
"dd_make_local_constraints called when there are no local constraints");
// ... and init_domdec_constraints always sets
// dd->constraint_comm...
"dd_make_local_constraints called when there are no local constraints");
// ... and init_domdec_constraints always sets
// dd->constraint_comm...
gmx::ArrayRef<const std::vector<int>> at2settle_mt;
/* When settle works inside charge groups, we assigned them already */
gmx::ArrayRef<const std::vector<int>> at2settle_mt;
/* When settle works inside charge groups, we assigned them already */
- if (dd->comm->systemInfo.haveSplitSettles)
+ if (dd->comm->systemInfo.mayHaveSplitSettles)
{
// TODO Perhaps gmx_domdec_constraints_t should keep a valid constr?
GMX_RELEASE_ASSERT(constr != nullptr, "Must have valid constraints object");
{
// TODO Perhaps gmx_domdec_constraints_t should keep a valid constr?
GMX_RELEASE_ASSERT(constr != nullptr, "Must have valid constraints object");
real cellsizeLimit = 0;
//! Can atoms connected by constraints be assigned to different domains?
real cellsizeLimit = 0;
//! Can atoms connected by constraints be assigned to different domains?
- bool haveSplitConstraints = false;
+ bool mayHaveSplitConstraints = false;
//! Can atoms connected by settles be assigned to different domains?
//! Can atoms connected by settles be assigned to different domains?
- bool haveSplitSettles = false;
+ bool mayHaveSplitSettles = false;
//! Estimated communication range needed for constraints
real constraintCommunicationRange = 0;
//! Estimated communication range needed for constraints
real constraintCommunicationRange = 0;
|| ddBondedChecking == DDBondedChecking::All,
"Invalid enum value for mdrun -ddcheck");
const ReverseTopOptions rtOptions(ddBondedChecking,
|| ddBondedChecking == DDBondedChecking::All,
"Invalid enum value for mdrun -ddcheck");
const ReverseTopOptions rtOptions(ddBondedChecking,
- !dd->comm->systemInfo.haveSplitConstraints,
- !dd->comm->systemInfo.haveSplitSettles);
+ !dd->comm->systemInfo.mayHaveSplitConstraints,
+ !dd->comm->systemInfo.mayHaveSplitSettles);
dd->reverse_top = std::make_unique<gmx_reverse_top_t>(
mtop, inputrec.efep != FreeEnergyPerturbationType::No, rtOptions);
dd->reverse_top = std::make_unique<gmx_reverse_top_t>(
mtop, inputrec.efep != FreeEnergyPerturbationType::No, rtOptions);
init_domdec_vsites(dd, numInterUpdategroupVirtualSites);
}
init_domdec_vsites(dd, numInterUpdategroupVirtualSites);
}
- if (dd->comm->systemInfo.haveSplitConstraints || dd->comm->systemInfo.haveSplitSettles)
+ if (dd->comm->systemInfo.mayHaveSplitConstraints || dd->comm->systemInfo.mayHaveSplitSettles)
{
init_domdec_constraints(dd, mtop);
}
{
init_domdec_constraints(dd, mtop);
}
}
break;
case DDAtomRanges::Type::Constraints:
}
break;
case DDAtomRanges::Type::Constraints:
- if (dd->comm->systemInfo.haveSplitConstraints || dd->comm->systemInfo.haveSplitSettles)
+ if (dd->comm->systemInfo.mayHaveSplitConstraints || dd->comm->systemInfo.mayHaveSplitSettles)
{
/* Only for inter-cg constraints we need special code */
n = dd_make_local_constraints(dd,
{
/* Only for inter-cg constraints we need special code */
n = dd_make_local_constraints(dd,
pull_t* pull_work,
FILE* log_p,
const t_commrec* cr_p,
pull_t* pull_work,
FILE* log_p,
const t_commrec* cr_p,
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle_p,
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle_p,
pull_t* pull_work,
FILE* log,
const t_commrec* cr,
pull_t* pull_work,
FILE* log,
const t_commrec* cr,
+ const bool useUpdateGroups,
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle,
bool pbcHandlingRequired,
int numConstraints,
int numSettles) :
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle,
bool pbcHandlingRequired,
int numConstraints,
int numSettles) :
- impl_(new Impl(mtop, ir, pull_work, log, cr, ms, nrnb, wcycle, pbcHandlingRequired, numConstraints, numSettles))
+ impl_(new Impl(mtop, ir, pull_work, log, cr, useUpdateGroups, ms, nrnb, wcycle, pbcHandlingRequired, numConstraints, numSettles))
pull_t* pull_work,
FILE* log_p,
const t_commrec* cr_p,
pull_t* pull_work,
FILE* log_p,
const t_commrec* cr_p,
+ const bool mayHaveSplitConstraints,
const gmx_multisim_t* ms_p,
t_nrnb* nrnb_p,
gmx_wallcycle* wcycle_p,
const gmx_multisim_t* ms_p,
t_nrnb* nrnb_p,
gmx_wallcycle* wcycle_p,
+ // When there are multiple PP domains and update groups are not in use,
+ // the constraints might be split across them.
if (ir.eConstrAlg == ConstraintAlgorithm::Lincs)
{
if (ir.eConstrAlg == ConstraintAlgorithm::Lincs)
{
- lincsd = init_lincs(log,
- mtop,
- nflexcon,
- at2con_mt,
- DOMAINDECOMP(cr) && ddHaveSplitConstraints(*cr->dd),
- ir.nLincsIter,
- ir.nProjOrder);
+ lincsd = init_lincs(
+ log, mtop, nflexcon, at2con_mt, mayHaveSplitConstraints, ir.nLincsIter, ir.nProjOrder);
}
if (ir.eConstrAlg == ConstraintAlgorithm::Shake)
{
}
if (ir.eConstrAlg == ConstraintAlgorithm::Shake)
{
- if (DOMAINDECOMP(cr) && ddHaveSplitConstraints(*cr->dd))
+ if (mayHaveSplitConstraints)
- "SHAKE is not supported with domain decomposition and constraint that "
+ "SHAKE is not supported with domain decomposition and constraints that "
"cross domain boundaries, use LINCS");
}
if (nflexcon)
"cross domain boundaries, use LINCS");
}
if (nflexcon)
pull_t* pull_work,
FILE* log,
const t_commrec* cr,
pull_t* pull_work,
FILE* log,
const t_commrec* cr,
+ bool haveHaveSplitConstraints,
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle,
const gmx_multisim_t* ms,
t_nrnb* nrnb,
gmx_wallcycle* wcycle,
// TODO This object will always return zero as RMSD value.
// It is more relevant to have non-zero value for testing.
constraints_ = makeConstraints(
// TODO This object will always return zero as RMSD value.
// It is more relevant to have non-zero value for testing.
constraints_ = makeConstraints(
- mtop_, inputrec_, nullptr, false, nullptr, &cr_, nullptr, nullptr, nullptr, false);
+ mtop_, inputrec_, nullptr, false, nullptr, &cr_, false, nullptr, nullptr, nullptr, false);
}
/*! \brief Helper function to generate synthetic data to output
}
/*! \brief Helper function to generate synthetic data to output
{
public:
//! Constructor, domdec should be nullptr without DD
{
public:
//! Constructor, domdec should be nullptr without DD
- Impl(const gmx_mtop_t& mtop, gmx_domdec_t* domdec, PbcType pbcType);
+ Impl(const gmx_mtop_t& mtop,
+ gmx_domdec_t* domdec,
+ PbcType pbcType,
+ ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType);
//! Returns the number of virtual sites acting over multiple update groups
int numInterUpdategroupVirtualSites() const { return numInterUpdategroupVirtualSites_; }
//! Returns the number of virtual sites acting over multiple update groups
int numInterUpdategroupVirtualSites() const { return numInterUpdategroupVirtualSites_; }
std::unique_ptr<VirtualSitesHandler> makeVirtualSitesHandler(const gmx_mtop_t& mtop,
const t_commrec* cr,
std::unique_ptr<VirtualSitesHandler> makeVirtualSitesHandler(const gmx_mtop_t& mtop,
const t_commrec* cr,
+ PbcType pbcType,
+ ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType)
{
GMX_RELEASE_ASSERT(cr != nullptr, "We need a valid commrec");
{
GMX_RELEASE_ASSERT(cr != nullptr, "We need a valid commrec");
- return std::make_unique<VirtualSitesHandler>(mtop, cr->dd, pbcType);
+ return std::make_unique<VirtualSitesHandler>(mtop, cr->dd, pbcType, updateGroupingPerMoleculeType);
}
ThreadingInfo::ThreadingInfo() : numThreads_(gmx_omp_nthreads_get(ModuleMultiThread::VirtualSite))
}
ThreadingInfo::ThreadingInfo() : numThreads_(gmx_omp_nthreads_get(ModuleMultiThread::VirtualSite))
-//! Returns the number of inter update-group vsites
-static int getNumInterUpdategroupVsites(const gmx_mtop_t& mtop, const gmx_domdec_t* domdec)
-{
- gmx::ArrayRef<const gmx::RangePartitioning> updateGroupingsPerMoleculeType;
- if (domdec)
- {
- updateGroupingsPerMoleculeType = getUpdateGroupingsPerMoleculeType(*domdec);
- }
-
- return countInterUpdategroupVsites(mtop, updateGroupingsPerMoleculeType);
-}
-
-VirtualSitesHandler::Impl::Impl(const gmx_mtop_t& mtop, gmx_domdec_t* domdec, const PbcType pbcType) :
- numInterUpdategroupVirtualSites_(getNumInterUpdategroupVsites(mtop, domdec)),
+VirtualSitesHandler::Impl::Impl(const gmx_mtop_t& mtop,
+ gmx_domdec_t* domdec,
+ const PbcType pbcType,
+ const ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType) :
+ numInterUpdategroupVirtualSites_(countInterUpdategroupVsites(mtop, updateGroupingPerMoleculeType)),
domainInfo_({ pbcType, pbcType != PbcType::No && numInterUpdategroupVirtualSites_ > 0, domdec }),
iparams_(mtop.ffparams.iparams)
{
}
domainInfo_({ pbcType, pbcType != PbcType::No && numInterUpdategroupVirtualSites_ > 0, domdec }),
iparams_(mtop.ffparams.iparams)
{
}
-VirtualSitesHandler::VirtualSitesHandler(const gmx_mtop_t& mtop, gmx_domdec_t* domdec, const PbcType pbcType) :
- impl_(new Impl(mtop, domdec, pbcType))
+VirtualSitesHandler::VirtualSitesHandler(const gmx_mtop_t& mtop,
+ gmx_domdec_t* domdec,
+ const PbcType pbcType,
+ const ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType) :
+ impl_(new Impl(mtop, domdec, pbcType, updateGroupingPerMoleculeType))
{
public:
//! Constructor, used only be the makeVirtualSitesHandler() factory function
{
public:
//! Constructor, used only be the makeVirtualSitesHandler() factory function
- VirtualSitesHandler(const gmx_mtop_t& mtop, gmx_domdec_t* domdec, PbcType pbcType);
+ VirtualSitesHandler(const gmx_mtop_t& mtop,
+ gmx_domdec_t* domdec,
+ PbcType pbcType,
+ ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType);
/*! \brief Create the virtual site handler
*
/*! \brief Create the virtual site handler
*
- * \param[in] mtop The global topology
- * \param[in] cr The communication record
- * \param[in] pbcType The type of PBC
+ * \param[in] mtop The global topology
+ * \param[in] cr The communication record
+ * \param[in] pbcType The type of PBC
+ * \param[in] updateGroupingPerMoleculeType Update grouping per molecule type, pass
+ * empty when not using update groups
* \returns A valid vsite handler object or nullptr when there are no virtual sites
*/
* \returns A valid vsite handler object or nullptr when there are no virtual sites
*/
-std::unique_ptr<VirtualSitesHandler> makeVirtualSitesHandler(const gmx_mtop_t& mtop,
- const t_commrec* cr,
- PbcType pbcType);
+std::unique_ptr<VirtualSitesHandler>
+makeVirtualSitesHandler(const gmx_mtop_t& mtop,
+ const t_commrec* cr,
+ PbcType pbcType,
+ ArrayRef<const RangePartitioning> updateGroupingPerMoleculeType);
}
/* Initialize the virtual site communication */
}
/* Initialize the virtual site communication */
- vsite = makeVirtualSitesHandler(mtop, cr, fr->pbcType);
+ gmx::ArrayRef<const gmx::RangePartitioning> updateGroupingsPerMoleculeType;
+ if (DOMAINDECOMP(cr))
+ {
+ updateGroupingsPerMoleculeType = getUpdateGroupingsPerMoleculeType(*cr->dd);
+ }
+ vsite = makeVirtualSitesHandler(mtop, cr, fr->pbcType, updateGroupingsPerMoleculeType);
calc_shifts(box, fr->shift_vec);
calc_shifts(box, fr->shift_vec);
}
/* Let makeConstraints know whether we have essential dynamics constraints. */
}
/* Let makeConstraints know whether we have essential dynamics constraints. */
- auto constr = makeConstraints(
- mtop, *inputrec, pull_work, doEssentialDynamics, fplog, cr, ms, &nrnb, wcycle.get(), fr->bMolPBC);
+ auto constr = makeConstraints(mtop,
+ *inputrec,
+ pull_work,
+ doEssentialDynamics,
+ fplog,
+ cr,
+ DOMAINDECOMP(cr) && ddMayHaveSplitConstraints(*cr->dd),
+ ms,
+ &nrnb,
+ wcycle.get(),
+ fr->bMolPBC);
/* Energy terms and groups */
gmx_enerdata_t enerd(mtop.groups.groups[SimulationAtomGroupType::EnergyOutput].size(),
/* Energy terms and groups */
gmx_enerdata_t enerd(mtop.groups.groups[SimulationAtomGroupType::EnergyOutput].size(),