struct gmx_domdec_t;
struct gmx_mtop_t;
+struct gmx_localtop_t;
struct t_commrec;
struct t_inputrec;
+class t_state;
namespace gmx
{
struct DomdecOptions;
struct MdrunOptions;
struct MDModulesNotifiers;
+class ObservablesReducerBuilder;
template<typename T>
class ArrayRef;
//! Destructor
~DomainDecompositionBuilder();
//! Build the resulting DD manager
- gmx_domdec_t* build(LocalAtomSetManager* atomSets);
+ gmx_domdec_t* build(LocalAtomSetManager* atomSets,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ ObservablesReducerBuilder* observablesReducerBuilder);
private:
class Impl;
bool directGpuCommUsedWithGpuUpdate);
//! Build the resulting DD manager
- gmx_domdec_t* build(LocalAtomSetManager* atomSets);
+ gmx_domdec_t* build(LocalAtomSetManager* atomSets,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ ObservablesReducerBuilder* observablesReducerBuilder);
//! Objects used in constructing and configuring DD
//! {
mdlog_, ddSettings_, options_.rankOrder, ddRankSetup_, cr_, ddCellIndex_, &pmeRanks_);
}
-gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets)
+gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ ObservablesReducerBuilder* observablesReducerBuilder)
{
gmx_domdec_t* dd = new gmx_domdec_t(ir_);
dd->atomSets = atomSets;
dd->localTopologyChecker = std::make_unique<LocalTopologyChecker>(
- mdlog_, cr_, mtop_, dd->comm->systemInfo.useUpdateGroups);
+ mdlog_, cr_, mtop_, localTopology, localState, dd->comm->systemInfo.useUpdateGroups, observablesReducerBuilder);
return dd;
}
{
}
-gmx_domdec_t* DomainDecompositionBuilder::build(LocalAtomSetManager* atomSets)
+gmx_domdec_t* DomainDecompositionBuilder::build(LocalAtomSetManager* atomSets,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ ObservablesReducerBuilder* observablesReducerBuilder)
{
- return impl_->build(atomSets);
+ return impl_->build(atomSets, localTopology, localState, observablesReducerBuilder);
}
DomainDecompositionBuilder::~DomainDecompositionBuilder() = default;
}
}
-const gmx::LocalTopologyChecker& dd_localTopologyChecker(const gmx_domdec_t& dd)
-{
- return *dd.localTopologyChecker;
-}
-
-gmx::LocalTopologyChecker* dd_localTopologyChecker(gmx_domdec_t* dd)
-{
- return dd->localTopologyChecker.get();
-}
-
void dd_init_local_state(const gmx_domdec_t& dd, const t_state* state_global, t_state* state_local)
{
std::array<int, 5> buf;
struct AtomInfoWithinMoleculeBlock;
class DeviceStreamManager;
class ForceWithShiftForces;
-class LocalTopologyChecker;
class MDLogger;
class RangePartitioning;
class VirtualSitesHandler;
*/
gmx::ArrayRef<const int> dd_constraints_nlocalatoms(const gmx_domdec_t* dd);
-/*! \brief Const getter for the local topology checker
- *
- * \returns Const handle to local topology checker */
-const gmx::LocalTopologyChecker& dd_localTopologyChecker(const gmx_domdec_t& dd);
-
-/*! \brief Getter for the local topology checker
- *
- * \returns Handle to local topology checker */
-gmx::LocalTopologyChecker* dd_localTopologyChecker(gmx_domdec_t* dd);
-
/*! \brief Construct local state */
void dd_init_local_state(const gmx_domdec_t& dd, const t_state* state_global, t_state* local_state);
#include "gromacs/domdec/reversetopology.h"
#include "gromacs/gmxlib/network.h"
#include "gromacs/mdtypes/commrec.h"
+#include "gromacs/mdtypes/observablesreducer.h"
+#include "gromacs/mdtypes/state.h"
#include "gromacs/topology/idef.h"
#include "gromacs/topology/ifunc.h"
#include "gromacs/topology/mtop_util.h"
const int numBondedInteractionsOverAllDomains,
const int expectedNumGlobalBondedInteractions,
const gmx_mtop_t& top_global,
- const gmx_localtop_t* top_local,
+ const gmx_localtop_t& top_local,
ArrayRef<const RVec> x,
const matrix box)
{
for (int ftype = 0; ftype < F_NRE; ftype++)
{
const int nral = NRAL(ftype);
- cl[ftype] = top_local->idef.il[ftype].size() / (1 + nral);
+ cl[ftype] = top_local.idef.il[ftype].size() / (1 + nral);
}
gmx_sumi(F_NRE, cl, cr);
}
}
- printMissingInteractionsAtoms(mdlog, cr, top_global, top_local->idef);
+ printMissingInteractionsAtoms(mdlog, cr, top_global, top_local.idef);
write_dd_pdb("dd_dump_err", 0, "dump", top_global, cr, -1, as_rvec_array(x.data()), box);
std::string errorMessage;
{
public:
//! Constructor
- Impl(const MDLogger& mdlog, const t_commrec* cr, const gmx_mtop_t& mtop, bool useUpdateGroups);
-
+ Impl(const MDLogger& mdlog,
+ const t_commrec* cr,
+ const gmx_mtop_t& mtop,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ bool useUpdateGroups);
//! Objects used when reporting that interactions are missing
//! {
//! Logger
const t_commrec* cr_;
//! Global system topology
const gmx_mtop_t& mtop_;
+ //! Local topology
+ const gmx_localtop_t& localTopology_;
+ //! Local state
+ const t_state& localState_;
//! }
- /*! \brief Number of bonded interactions found in the local
- * topology for this domain. */
- int numBondedInteractionsToReduce_ = 0;
- /*! \brief Whether to check at the next global communication
- * stage the total number of bonded interactions found.
- *
- * Cleared after that number is found. */
- bool shouldCheckNumberOfBondedInteractions_ = false;
- /*! \brief The total number of bonded interactions found in
- * the local topology across all domains.
+ /*! \brief View used for computing the global number of bonded interactions.
*
- * Only has a value after reduction across all ranks, which is
- * removed when it is again time to check after a new
- * partition. */
- std::optional<int> numBondedInteractionsOverAllDomains_;
- //! The number of bonded interactions computed from the full system topology
- int expectedNumGlobalBondedInteractions_ = 0;
+ * Can be written any time, but that is only useful when followed
+ * by a call of the callbackToRequireReduction. Useful to read
+ * only from the callback that the ObservablesReducer will later
+ * make after reduction. */
+ gmx::ArrayRef<double> reductionBuffer_;
+ /*! \brief Callback used after repartitioning to require reduction
+ * of numBondedInteractionsToReduce so that the total number of
+ * bonded interactions can be checked. */
+ gmx::ObservablesReducerBuilder::CallbackToRequireReduction callbackToRequireReduction_;
+ /*! \brief The expected number of global bonded interactions from the system topology */
+ int expectedNumGlobalBondedInteractions_;
};
return expectedNumGlobalBondedInteractions;
}
-LocalTopologyChecker::Impl::Impl(const MDLogger& mdlog,
- const t_commrec* cr,
- const gmx_mtop_t& mtop,
- const bool useUpdateGroups) :
+LocalTopologyChecker::Impl::Impl(const MDLogger& mdlog,
+ const t_commrec* cr,
+ const gmx_mtop_t& mtop,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ bool useUpdateGroups) :
mdlog_(mdlog),
cr_(cr),
mtop_(mtop),
+ localTopology_(localTopology),
+ localState_(localState),
expectedNumGlobalBondedInteractions_(computeExpectedNumGlobalBondedInteractions(mtop, useUpdateGroups))
{
}
-LocalTopologyChecker::LocalTopologyChecker(const MDLogger& mdlog,
- const t_commrec* cr,
- const gmx_mtop_t& mtop,
- const bool useUpdateGroups) :
- impl_(std::make_unique<Impl>(mdlog, cr, mtop, useUpdateGroups))
+LocalTopologyChecker::LocalTopologyChecker(const MDLogger& mdlog,
+ const t_commrec* cr,
+ const gmx_mtop_t& mtop,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ const bool useUpdateGroups,
+ ObservablesReducerBuilder* observablesReducerBuilder) :
+ impl_(std::make_unique<Impl>(mdlog, cr, mtop, localTopology, localState, useUpdateGroups))
{
+ Impl* impl = impl_.get();
+ ObservablesReducerBuilder::CallbackFromBuilder callbackFromBuilder =
+ [impl](ObservablesReducerBuilder::CallbackToRequireReduction c, gmx::ArrayRef<double> v) {
+ impl->callbackToRequireReduction_ = std::move(c);
+ impl->reductionBuffer_ = v;
+ };
+
+ // Make the callback that runs afer reduction.
+ ObservablesReducerBuilder::CallbackAfterReduction callbackAfterReduction = [impl](gmx::Step /*step*/) {
+ // Get the total after reduction
+ int numTotalBondedInteractionsFound = impl->reductionBuffer_[0];
+ if (numTotalBondedInteractionsFound != impl->expectedNumGlobalBondedInteractions_)
+ {
+ // Give error and exit
+ dd_print_missing_interactions(impl->mdlog_,
+ impl->cr_,
+ numTotalBondedInteractionsFound,
+ impl->expectedNumGlobalBondedInteractions_,
+ impl->mtop_,
+ impl->localTopology_,
+ impl->localState_.x,
+ impl->localState_.box); // Does not return
+ }
+ };
+
+ observablesReducerBuilder->addSubscriber(
+ 1, std::move(callbackFromBuilder), std::move(callbackAfterReduction));
}
LocalTopologyChecker::~LocalTopologyChecker() = default;
void LocalTopologyChecker::scheduleCheckOfLocalTopology(const int numBondedInteractionsToReduce)
{
- impl_->numBondedInteractionsToReduce_ = numBondedInteractionsToReduce;
- // Note that it's possible for this to still be true from the last
- // time it was set, e.g. if repartitioning was triggered before
- // global communication that would have acted on the true
- // value. This could happen for example when replica exchange took
- // place soon after a partition.
- impl_->shouldCheckNumberOfBondedInteractions_ = true;
- // Clear the old global value, which is now invalid
- impl_->numBondedInteractionsOverAllDomains_.reset();
-}
-
-bool LocalTopologyChecker::shouldCheckNumberOfBondedInteractions() const
-{
- return impl_->shouldCheckNumberOfBondedInteractions_;
-}
-
-int LocalTopologyChecker::numBondedInteractions() const
-{
- return impl_->numBondedInteractionsToReduce_;
-}
-
-void LocalTopologyChecker::setNumberOfBondedInteractionsOverAllDomains(const int newValue)
-{
- GMX_RELEASE_ASSERT(!impl_->numBondedInteractionsOverAllDomains_.has_value(),
- "Cannot set number of bonded interactions because it is already set");
- impl_->numBondedInteractionsOverAllDomains_.emplace(newValue);
-}
-
-void LocalTopologyChecker::checkNumberOfBondedInteractions(const gmx_localtop_t* top_local,
- ArrayRef<const RVec> x,
- const matrix box)
-{
- if (impl_->shouldCheckNumberOfBondedInteractions_)
- {
- GMX_RELEASE_ASSERT(impl_->numBondedInteractionsOverAllDomains_.has_value(),
- "The check for the total number of bonded interactions requires the "
- "value to have been reduced across all domains");
- if (impl_->numBondedInteractionsOverAllDomains_.value() != impl_->expectedNumGlobalBondedInteractions_)
- {
- dd_print_missing_interactions(impl_->mdlog_,
- impl_->cr_,
- impl_->numBondedInteractionsOverAllDomains_.value(),
- impl_->expectedNumGlobalBondedInteractions_,
- impl_->mtop_,
- top_local,
- x,
- box); // Does not return
- }
- // Now that the value is set and the check complete, future
- // global communication should not compute the value until
- // after the next partitioning.
- impl_->shouldCheckNumberOfBondedInteractions_ = false;
- }
+ // Fill the reduction buffer with the value from this domain to reduce
+ impl_->reductionBuffer_[0] = double(numBondedInteractionsToReduce);
+
+ // Pass the post-reduction callback to the ObservablesReducer via
+ // the callback it gave us for the purpose.
+ //
+ // Note that it's possible that the callbackAfterReduction is already
+ // outstanding, e.g. if repartitioning was triggered before
+ // observables were reduced. This could happen for example when
+ // replica exchange took place soon after a partition. If so, the
+ // callback will be called again. So long as there is no race
+ // between the calls to this function and the calls to
+ // ObservablesReducer for reduction, this will work correctly. It
+ // could be made safer e.g. with checks against duplicate
+ // callbacks, but there is no problem to solve.
+ //
+ // There is no need to check the return value from this callback,
+ // as it is not an error to request reduction at a future step.
+ impl_->callbackToRequireReduction_(ReductionRequirement::Eventually);
}
} // namespace gmx
struct gmx_mtop_t;
struct t_commrec;
struct t_inputrec;
+class t_state;
namespace gmx
{
class MDLogger;
-template<typename>
-class ArrayRef;
+class ObservablesReducerBuilder;
} // namespace gmx
namespace gmx
{
-/*! \brief Has responsibility for checking that the local topology
- * distributed across domains describes a total number of bonded
- * interactions that matches the system topology
+/*! \libinternal
+ * \brief Has responsibility for checking that the local topology distributed
+ * across domains describes a total number of bonded interactions that matches
+ * the system topology
*
- * Because this check is not urgent, the communication that it
- * requires is done at the next opportunity, rather than requiring
- * extra communication. If the check fails, a fatal error stops
- * execution. In principle, if there was a bug, GROMACS might crash in
- * the meantime because of the wrong forces. However as a bug is
- * unlikely, we optimize by avoiding creating extra overhead from
- * communication.
+ * This uses the ObservablesReducer framework to check that the count
+ * of bonded interactions in the local topology made for each domain
+ * sums to the expected value. Because this check is not urgent, the
+ * communication that it requires is done at the next opportunity,
+ * rather than requiring extra communication. If the check fails, a
+ * fatal error stops execution. In principle, if there was a bug,
+ * GROMACS might crash in the meantime because of the wrong
+ * forces. However as a bug is unlikely we optimize by avoiding
+ * creating extra overhead from communication.
*/
class LocalTopologyChecker
{
* \param[in] mdlog Logger
* \param[in] cr Communication object
* \param[in] mtop Global system topology
+ * \param[in] localTopology The local topology
+ * \param[in] localState The local state
* \param[in] useUpdateGroups Whether update groups are in use
+ * \param[in] observablesReducerBuilder Handle to builder for ObservablesReducer
*/
- LocalTopologyChecker(const MDLogger& mdlog, const t_commrec* cr, const gmx_mtop_t& mtop, bool useUpdateGroups);
+ LocalTopologyChecker(const MDLogger& mdlog,
+ const t_commrec* cr,
+ const gmx_mtop_t& mtop,
+ const gmx_localtop_t& localTopology,
+ const t_state& localState,
+ bool useUpdateGroups,
+ ObservablesReducerBuilder* observablesReducerBuilder);
//! Destructor
~LocalTopologyChecker();
//! Move constructor
* another module. */
void scheduleCheckOfLocalTopology(int numBondedInteractionsToReduce);
- /*! \brief Return whether the total bonded interaction count across
- * domains should be checked in observables reduction this step. */
- bool shouldCheckNumberOfBondedInteractions() const;
-
- //! Return the number of bonded interactions in this domain.
- int numBondedInteractions() const;
-
- /*! \brief Set total bonded interaction count across domains. */
- void setNumberOfBondedInteractionsOverAllDomains(int newValue);
-
- /*! \brief Check whether bonded interactions are missing from the reverse topology
- * produced by domain decomposition.
- *
- * \param[in] top_local Local topology for the error message
- * \param[in] x Position vector for the error message
- * \param[in] box Box matrix for the error message
- */
- void checkNumberOfBondedInteractions(const gmx_localtop_t* top_local,
- ArrayRef<const RVec> x,
- const matrix box);
-
private:
class Impl;
std::unique_ptr<Impl> impl_;
{
gmx_bool bEner, bPres, bTemp;
gmx_bool bStopCM, bGStat, bReadEkin, bEkinAveVel, bScaleEkin, bConstrain;
- gmx_bool bCheckNumberOfBondedInteractions;
real dvdl_ekin;
/* translate CGLO flags to gmx_booleans */
- bStopCM = ((flags & CGLO_STOPCM) != 0);
- bGStat = ((flags & CGLO_GSTAT) != 0);
- bReadEkin = ((flags & CGLO_READEKIN) != 0);
- bScaleEkin = ((flags & CGLO_SCALEEKIN) != 0);
- bEner = ((flags & CGLO_ENERGY) != 0);
- bTemp = ((flags & CGLO_TEMPERATURE) != 0);
- bPres = ((flags & CGLO_PRESSURE) != 0);
- bConstrain = ((flags & CGLO_CONSTRAINT) != 0);
- bCheckNumberOfBondedInteractions = ((flags & CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS) != 0);
+ bStopCM = ((flags & CGLO_STOPCM) != 0);
+ bGStat = ((flags & CGLO_GSTAT) != 0);
+ bReadEkin = ((flags & CGLO_READEKIN) != 0);
+ bScaleEkin = ((flags & CGLO_SCALEEKIN) != 0);
+ bEner = ((flags & CGLO_ENERGY) != 0);
+ bTemp = ((flags & CGLO_TEMPERATURE) != 0);
+ bPres = ((flags & CGLO_PRESSURE) != 0);
+ bConstrain = ((flags & CGLO_CONSTRAINT) != 0);
/* we calculate a full state kinetic energy either with full-step velocity verlet
or half step where we need the pressure */
calc_vcm_grp(*mdatoms, x, v, vcm);
}
- if (bTemp || bStopCM || bPres || bEner || bConstrain || bCheckNumberOfBondedInteractions
+ if (bTemp || bStopCM || bPres || bEner || bConstrain
|| !observablesReducer->communicationBuffer().empty())
{
if (!bGStat)
#define CGLO_READEKIN (1u << 10u)
/* we need to reset the ekin rescaling factor here */
#define CGLO_SCALEEKIN (1u << 11u)
-/* After a new DD partitioning, we need to set a flag to schedule
- * global reduction of the total number of bonded interactions that
- * will be computed, to check none are missing. */
-#define CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS (1u << 12u)
/*! \brief Return the number of steps that will take place between
* intra-simulation communications, given the constraints of the
#include "gromacs/domdec/domdec.h"
#include "gromacs/domdec/domdec_struct.h"
-#include "gromacs/domdec/localtopologychecker.h"
#include "gromacs/fileio/checkpoint.h"
#include "gromacs/fileio/xtcio.h"
#include "gromacs/gmxlib/network.h"
/* instead of current system, gmx_booleans for summing virial, kinetic energy, and other terms */
{
int ie = 0, ifv = 0, isv = 0, irmsd = 0;
- int idedl = 0, idedlo = 0, idvdll = 0, idvdlnl = 0, iepl = 0, icm = 0, imass = 0, ica = 0, inb = 0;
+ int idedl = 0, idedlo = 0, idvdll = 0, idvdlnl = 0, iepl = 0, icm = 0, imass = 0, ica = 0;
int isig = -1;
int icj = -1, ici = -1, icx = -1;
- bool checkNumberOfBondedInteractions = (flags & CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS) != 0;
- bool bVV = EI_VV(inputrec.eI);
- bool bTemp = ((flags & CGLO_TEMPERATURE) != 0);
- bool bEner = ((flags & CGLO_ENERGY) != 0);
- bool bPres = ((flags & CGLO_PRESSURE) != 0);
- bool bConstrVir = ((flags & CGLO_CONSTRAINT) != 0);
- bool bEkinAveVel = (inputrec.eI == IntegrationAlgorithm::VV
+ bool bVV = EI_VV(inputrec.eI);
+ bool bTemp = ((flags & CGLO_TEMPERATURE) != 0);
+ bool bEner = ((flags & CGLO_ENERGY) != 0);
+ bool bPres = ((flags & CGLO_PRESSURE) != 0);
+ bool bConstrVir = ((flags & CGLO_CONSTRAINT) != 0);
+ bool bEkinAveVel = (inputrec.eI == IntegrationAlgorithm::VV
|| (inputrec.eI == IntegrationAlgorithm::VVAK && bPres));
- bool bReadEkin = ((flags & CGLO_READEKIN) != 0);
+ bool bReadEkin = ((flags & CGLO_READEKIN) != 0);
// This structure implements something akin to a vector. As
// modules add their data into it with add_bin[rd], they save the
}
}
- double nb;
- if (checkNumberOfBondedInteractions)
- {
- GMX_RELEASE_ASSERT(DOMAINDECOMP(cr),
- "No need to check number of bonded interactions when not using domain "
- "decomposition");
- nb = cr->dd->localTopologyChecker->numBondedInteractions();
- inb = add_bind(rb, 1, &nb);
- }
if (!sig.empty())
{
isig = add_binr(rb, sig);
}
}
- if (checkNumberOfBondedInteractions)
- {
- extract_bind(rb, inb, 1, &nb);
- GMX_RELEASE_ASSERT(DOMAINDECOMP(cr),
- "No need to check number of bonded interactions when not using domain "
- "decomposition");
- cr->dd->localTopologyChecker->setNumberOfBondedInteractionsOverAllDomains(gmx::roundToInt(nb));
- }
-
if (!sig.empty())
{
extract_binr(rb, isig, sig);
#include <algorithm>
-#include "gromacs/domdec/domdec.h"
#include "gromacs/domdec/localtopologychecker.h"
#include "gromacs/gmxlib/nrnb.h"
#include "gromacs/math/vec.h"
t_fcdata* fcdata,
t_extmass* MassQ,
t_vcm* vcm,
- const gmx_localtop_t& top,
gmx_enerdata_t* enerd,
gmx::ObservablesReducer* observablesReducer,
gmx_ekindata_t* ekind,
((bGStat ? CGLO_GSTAT : 0) | (bCalcEner ? CGLO_ENERGY : 0)
| (bTemp ? CGLO_TEMPERATURE : 0) | (bPres ? CGLO_PRESSURE : 0)
| (bPres ? CGLO_CONSTRAINT : 0) | (bStopCM ? CGLO_STOPCM : 0) | CGLO_SCALEEKIN);
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat,
cr,
ir,
time step kinetic energy for the pressure (always true now, since we want accurate statistics).
b) If we are using EkinAveEkin for the kinetic energy for the temperature control, we still feed in
EkinAveVel because it's needed for the pressure */
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
if (bStopCM)
{
process_and_stopcm_grp(
class gmx_ekindata_t;
struct gmx_enerdata_t;
struct gmx_global_stat;
-struct gmx_localtop_t;
-struct gmx_mtop_t;
struct gmx_wallcycle;
struct pull_t;
struct t_commrec;
{
class Constraints;
class ForceBuffers;
-class MDLogger;
class ObservablesReducer;
class SimulationSignaller;
class Update;
* \param[in] fcdata Force calculation data.
* \param[in] MassQ Mass/pressure data.
* \param[in] vcm Center of mass motion removal.
- * \param[in] top Local topology.
* \param[in] enerd Energy data.
* \param[in] observablesReducer Pointer to the \c ObservablesReducer object
* \param[in] ekind Kinetic energy data.
t_fcdata* fcdata,
t_extmass* MassQ,
t_vcm* vcm,
- const gmx_localtop_t& top,
gmx_enerdata_t* enerd,
gmx::ObservablesReducer* observablesReducer,
gmx_ekindata_t* ekind,
class gmx_ekindata_t;
struct gmx_enerdata_t;
struct gmx_enfrot;
+struct gmx_localtop_t;
struct gmx_mtop_t;
struct gmx_membed_t;
struct gmx_multisim_t;
pull_t* pull_work,
t_swap* swap,
const gmx_mtop_t& top_global,
+ gmx_localtop_t* top,
t_state* state_global,
+ t_state* state,
ObservablesHistory* observablesHistory,
MDAtoms* mdAtoms,
t_nrnb* nrnb,
pull_work(pull_work),
swap(swap),
top_global(top_global),
+ top(top),
state_global(state_global),
+ state(state),
observablesHistory(observablesHistory),
mdAtoms(mdAtoms),
nrnb(nrnb),
t_swap* swap;
//! Full system topology.
const gmx_mtop_t& top_global;
+ //! Handle to local simulation topology.
+ gmx_localtop_t* top;
//! Full simulation state (only non-nullptr on master rank).
t_state* state_global;
+ //! Handle to local state of the simulation.
+ t_state* state;
//! History of simulation observables.
ObservablesHistory* observablesHistory;
//! Atom parameters for this domain.
}
}
- // Local state only becomes valid now.
- std::unique_ptr<t_state> stateInstance;
- t_state* state;
-
- gmx_localtop_t top(top_global.ffparams);
ObservablesReducer observablesReducer = observablesReducerBuilder->build();
ForceBuffers f(simulationWork.useMts,
const t_mdatoms* md = mdAtoms->mdatoms();
if (DOMAINDECOMP(cr))
{
- stateInstance = std::make_unique<t_state>();
- state = stateInstance.get();
+ // Local state only becomes valid now.
dd_init_local_state(*cr->dd, state_global, state);
/* Distribute the charge groups over the nodes from the master node */
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
else
{
state_change_natoms(state_global, state_global->natoms);
- /* Copy the pointer to the global state */
- state = state_global;
/* Generate and initialize new topology */
- mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc);
+ mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc);
upd.updateAfterPartition(state->natoms,
md->cFREEZE ? gmx::arrayRefFromArray(md->cFREEZE, md->nr)
cglo_flags_iteration |= CGLO_STOPCM;
cglo_flags_iteration &= ~CGLO_TEMPERATURE;
}
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()
- && cgloIteration == 0)
- {
- cglo_flags_iteration |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat,
cr,
ir,
inc_nrnb(nrnb, eNR_STOPCM, md->homenr);
}
}
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
if (ir->eI == IntegrationAlgorithm::VVAK)
{
/* a second call to get the half step temperature initialized as well */
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
* the full step kinetic energy and possibly for T-coupling.*/
/* This may not be quite working correctly yet . . . . */
int cglo_flags = CGLO_GSTAT | CGLO_TEMPERATURE;
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat,
cr,
ir,
cglo_flags,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
}
clear_mat(force_vir);
pull_work,
bNS,
force_flags,
- &top,
+ top,
constr,
enerd,
state->natoms,
step,
nrnb,
wcycle,
- &top,
+ top,
state->box,
state->x.arrayRefWithPadding(),
&state->hist,
&fcdata,
&MassQ,
&vcm,
- top,
enerd,
&observablesReducer,
ekind,
integrator->set(stateGpu->getCoordinates(),
stateGpu->getVelocities(),
stateGpu->getForces(),
- top.idef,
+ top->idef,
*md);
// Copy data to the GPU after buffers might have being reinitialized
(bGStat ? CGLO_GSTAT : 0) | (!EI_VV(ir->eI) && bCalcEner ? CGLO_ENERGY : 0)
| (!EI_VV(ir->eI) && bStopCM ? CGLO_STOPCM : 0)
| (!EI_VV(ir->eI) ? CGLO_TEMPERATURE : 0)
- | (!EI_VV(ir->eI) ? CGLO_PRESSURE : 0) | CGLO_CONSTRAINT
- | (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()
- ? CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS
- : 0),
+ | (!EI_VV(ir->eI) ? CGLO_PRESSURE : 0) | CGLO_CONSTRAINT,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
if (!EI_VV(ir->eI) && bStopCM)
{
process_and_stopcm_grp(
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
}
}
- // Local state only becomes valid now.
- std::unique_ptr<t_state> stateInstance;
- t_state* state;
-
- gmx_localtop_t top(top_global.ffparams);
-
if (DOMAINDECOMP(cr))
{
- stateInstance = std::make_unique<t_state>();
- state = stateInstance.get();
+ // Local state only becomes valid now.
dd_init_local_state(*cr->dd, state_global, state);
/* Distribute the charge groups over the nodes from the master node */
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
else
{
state_change_natoms(state_global, state_global->natoms);
- /* Copy the pointer to the global state */
- state = state_global;
-
- mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc);
+ mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc);
}
auto* mdatoms = mdAtoms->mdatoms();
int64_t step_rel = 0;
{
- int cglo_flags = CGLO_GSTAT;
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
+ int cglo_flags = CGLO_GSTAT;
bool bSumEkinhOld = false;
t_vcm* vcm = nullptr;
compute_globals(gstat,
cglo_flags,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
}
if (MASTER(cr))
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
pull_work,
bNS,
force_flags,
- &top,
+ top,
constr,
enerd,
state->natoms,
step,
nrnb,
wcycle,
- &top,
+ top,
state->box,
state->x.arrayRefWithPadding(),
&state->hist,
SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal);
int cglo_flags = CGLO_GSTAT | CGLO_ENERGY;
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat,
cr,
ir,
cglo_flags,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
}
{
if (DOMAINDECOMP(cr))
{
+ // Local state only becomes valid now.
dd_init_local_state(*cr->dd, state_global, &ems->s);
/* Distribute the charge groups over the nodes from the master node */
{
const char* CG = "Polak-Ribiere Conjugate Gradients";
- gmx_localtop_t top(top_global.ffparams);
gmx_global_stat_t gstat;
double tmp, minstep;
real stepsize;
state_global,
top_global,
s_min,
- &top,
+ top,
nrnb,
fr,
mdAtoms,
cr,
ms,
top_global,
- &top,
+ top,
inputrec,
imdSession,
pull_work,
imdSession,
pull_work,
s_min,
- &top,
+ top,
mdAtoms,
fr,
vsite,
imdSession,
pull_work,
s_min,
- &top,
+ top,
mdAtoms,
fr,
vsite,
{
static const char* LBFGS = "Low-Memory BFGS Minimizer";
em_state_t ems;
- gmx_localtop_t top(top_global.ffparams);
gmx_global_stat_t gstat;
auto* mdatoms = mdAtoms->mdatoms();
state_global,
top_global,
&ems,
- &top,
+ top,
nrnb,
fr,
mdAtoms,
cr,
ms,
top_global,
- &top,
+ top,
inputrec,
imdSession,
pull_work,
void LegacySimulator::do_steep()
{
const char* SD = "Steepest Descents";
- gmx_localtop_t top(top_global.ffparams);
gmx_global_stat_t gstat;
real stepsize;
real ustep;
state_global,
top_global,
s_try,
- &top,
+ top,
nrnb,
fr,
mdAtoms,
cr,
ms,
top_global,
- &top,
+ top,
inputrec,
imdSession,
pull_work,
imdSession,
pull_work,
s_min,
- &top,
+ top,
mdAtoms,
fr,
vsite,
{
const char* NM = "Normal Mode Analysis";
int nnodes;
- gmx_localtop_t top(top_global.ffparams);
gmx_global_stat_t gstat;
tensor vir, pres;
rvec mu_tot = { 0 };
state_global,
top_global,
&state_work,
- &top,
+ top,
nrnb,
fr,
mdAtoms,
cr,
ms,
top_global,
- &top,
+ top,
inputrec,
imdSession,
pull_work,
pull_work,
bNS,
force_flags,
- &top,
+ top,
constr,
enerd,
state_work.s.natoms,
t_trxstatus* status = nullptr;
rvec mu_tot;
t_trxframe rerun_fr;
- gmx_localtop_t top(top_global.ffparams);
ForceBuffers f;
gmx_global_stat_t gstat;
gmx_shellfc_t* shellfc;
}
}
- // Local state only becomes valid now.
- std::unique_ptr<t_state> stateInstance;
- t_state* state;
-
if (DOMAINDECOMP(cr))
{
- stateInstance = std::make_unique<t_state>();
- state = stateInstance.get();
+ // Local state only becomes valid now.
dd_init_local_state(*cr->dd, state_global, state);
/* Distribute the charge groups over the nodes from the master node */
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
/* Copy the pointer to the global state */
state = state_global;
- mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc);
+ mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc);
}
auto* mdatoms = mdAtoms->mdatoms();
int64_t step_rel = 0;
{
- int cglo_flags = CGLO_GSTAT;
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
+ int cglo_flags = CGLO_GSTAT;
bool bSumEkinhOld = false;
t_vcm* vcm = nullptr;
compute_globals(gstat,
cglo_flags,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
}
if (MASTER(cr))
state,
&f,
mdAtoms,
- &top,
+ top,
fr,
vsite,
constr,
pull_work,
bNS,
force_flags,
- &top,
+ top,
constr,
enerd,
state->natoms,
step,
nrnb,
wcycle,
- &top,
+ top,
state->box,
state->x.arrayRefWithPadding(),
&state->hist,
SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal);
int cglo_flags = CGLO_GSTAT | CGLO_ENERGY;
- if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions())
- {
- cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat,
cr,
ir,
cglo_flags,
step,
&observablesReducer);
- if (DOMAINDECOMP(cr))
- {
- dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions(
- &top, makeConstArrayRef(state->x), state->box);
- }
}
/* Note: this is OK, but there are some numerical precision issues with using the convergence of
// requires it (e.g. pull, CompEl, density fitting), so that we
// don't update the local atom sets unilaterally every step.
LocalAtomSetManager atomSets;
+
+ // Local state and topology are declared (and perhaps constructed)
+ // now, because DD needs them for the LocalTopologyChecker, but
+ // they do not contain valid data until after the first DD
+ // partition.
+ std::unique_ptr<t_state> localStateInstance;
+ t_state* localState;
+ gmx_localtop_t localTopology(mtop.ffparams);
+
if (ddBuilder)
{
+ localStateInstance = std::make_unique<t_state>();
+ localState = localStateInstance.get();
// TODO Pass the GPU streams to ddBuilder to use in buffer
// transfers (e.g. halo exchange)
- cr->dd = ddBuilder->build(&atomSets);
+ cr->dd = ddBuilder->build(&atomSets, localTopology, *localState, &observablesReducerBuilder);
// The builder's job is done, so destruct it
ddBuilder.reset(nullptr);
// Note that local state still does not exist yet.
}
+ else
+ {
+ // Without DD, the local state is merely an alias to the global state,
+ // so we don't need to allocate anything.
+ localState = globalState.get();
+ }
+
// Ensure that all atoms within the same update group are in the
// same periodic image. Otherwise, a simulation that did not use
// update groups (e.g. a single-rank simulation) cannot always be
GMX_ASSERT(stopHandlerBuilder_, "Runner must provide StopHandlerBuilder to simulator.");
SimulatorBuilder simulatorBuilder;
- simulatorBuilder.add(SimulatorStateData(globalState.get(), &observablesHistory, &enerd, &ekind));
+ simulatorBuilder.add(SimulatorStateData(
+ globalState.get(), localState, &observablesHistory, &enerd, &ekind));
simulatorBuilder.add(std::move(membedHolder));
simulatorBuilder.add(std::move(stopHandlerBuilder_));
simulatorBuilder.add(SimulatorConfig(mdrunOptions, startingBehavior, &runScheduleWork));
simulatorBuilder.add(CenterOfMassPulling(pull_work));
// Todo move to an MDModule
simulatorBuilder.add(IonSwapping(swap));
- simulatorBuilder.add(TopologyData(mtop, mdAtoms.get()));
+ simulatorBuilder.add(TopologyData(mtop, &localTopology, mdAtoms.get()));
simulatorBuilder.add(BoxDeformationHandle(deform.get()));
simulatorBuilder.add(std::move(modularSimulatorCheckpointData));
// As soon as we destroy GPU contexts after mdrunner() exits, these lines should go.
mdAtoms.reset(nullptr);
globalState.reset(nullptr);
+ localStateInstance.reset(nullptr);
mdModules_.reset(nullptr); // destruct force providers here as they might also use the GPU
fr.reset(nullptr); // destruct forcerec before gpu
// TODO convert to C++ so we can get rid of these frees
centerOfMassPulling_->pull_work,
ionSwapping_->ionSwap,
topologyData_->top_global,
+ topologyData_->localTopology,
simulatorStateData_->globalState_p,
+ simulatorStateData_->localState_p,
simulatorStateData_->observablesHistory_p,
topologyData_->mdAtoms,
profiling_->nrnb,
centerOfMassPulling_->pull_work,
ionSwapping_->ionSwap,
topologyData_->top_global,
+ topologyData_->localTopology,
simulatorStateData_->globalState_p,
+ simulatorStateData_->localState_p,
simulatorStateData_->observablesHistory_p,
topologyData_->mdAtoms,
profiling_->nrnb,
class gmx_ekindata_t;
struct gmx_enerdata_t;
struct gmx_enfrot;
+struct gmx_localtop_t;
struct gmx_mtop_t;
struct gmx_multisim_t;
struct gmx_output_env_t;
{
//! Build collection of current state data.
SimulatorStateData(t_state* globalState,
+ t_state* localState,
ObservablesHistory* observablesHistory,
gmx_enerdata_t* enerdata,
gmx_ekindata_t* ekindata) :
globalState_p(globalState),
+ localState_p(localState),
observablesHistory_p(observablesHistory),
enerdata_p(enerdata),
ekindata_p(ekindata)
//! Handle to global state of the simulation.
t_state* globalState_p;
+ //! Handle to local state of the simulation.
+ t_state* localState_p;
//! Handle to current simulation history.
ObservablesHistory* observablesHistory_p;
//! Handle to collected data for energy groups.
{
public:
//! Build collection from simulation data.
- TopologyData(const gmx_mtop_t& globalTopology, MDAtoms* mdAtoms) :
- top_global(globalTopology), mdAtoms(mdAtoms)
+ TopologyData(const gmx_mtop_t& globalTopology, gmx_localtop_t* localTopology, MDAtoms* mdAtoms) :
+ top_global(globalTopology), localTopology(localTopology), mdAtoms(mdAtoms)
{
}
//! Handle to global simulation topology.
const gmx_mtop_t& top_global;
+ //! Handle to local simulation topology.
+ gmx_localtop_t* localTopology;
//! Handle to information about MDAtoms.
MDAtoms* mdAtoms;
};
GMX_RELEASE_ASSERT(gmx_omp_nthreads_get(ModuleMultiThread::Default) == 1,
"TPI does not support OpenMP");
- gmx_localtop_t top(top_global.ffparams);
gmx::ForceBuffers f;
real lambda, t, temp, beta, drmax, epot;
double embU, sum_embU, *sum_UgembU, V, V_all, VembU_all;
nnodes = cr->nnodes;
- gmx_mtop_generate_local_top(top_global, &top, inputrec->efep != FreeEnergyPerturbationType::No);
+ gmx_mtop_generate_local_top(top_global, top, inputrec->efep != FreeEnergyPerturbationType::No);
const SimulationGroups* groups = &top_global.groups;
gmx::constArrayRefFromArray(mdatoms->chargeA, mdatoms->nr),
fr->atomInfo);
- fr->nbv->constructPairlist(InteractionLocality::Local, top.excls, step, nrnb);
+ fr->nbv->constructPairlist(InteractionLocality::Local, top->excls, step, nrnb);
bNS = FALSE;
}
step,
nrnb,
wcycle,
- &top,
+ top,
state_global->box,
state_global->x.arrayRefWithPadding(),
&state_global->hist,
#include "computeglobalselement.h"
#include "gromacs/domdec/domdec.h"
-#include "gromacs/domdec/localtopologychecker.h"
#include "gromacs/gmxlib/network.h"
#include "gromacs/gmxlib/nrnb.h"
#include "gromacs/math/vec.h"
nullSignaller_(std::make_unique<SimulationSignaller>(nullptr, nullptr, nullptr, false, false)),
statePropagatorData_(statePropagatorData),
energyData_(energyData),
- localTopology_(nullptr),
freeEnergyPerturbationData_(freeEnergyPerturbationData),
vcm_(global_top.groups, *inputrec),
signals_(signals),
template<ComputeGlobalsAlgorithm algorithm>
void ComputeGlobalsElement<algorithm>::elementSetup()
{
- GMX_ASSERT(localTopology_, "Setup called before local topology was set.");
-
if (doStopCM_ && !inputrec_->bContinuation)
{
// To minimize communication, compute_globals computes the COM velocity
const auto* lastbox = useLastBox ? statePropagatorData_->constPreviousBox()
: statePropagatorData_->constBox();
- if (DOMAINDECOMP(cr_) && dd_localTopologyChecker(*cr_->dd).shouldCheckNumberOfBondedInteractions())
- {
- flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
- }
compute_globals(gstat_,
cr_,
inputrec_,
flags,
step,
observablesReducer_);
- if (DOMAINDECOMP(cr_))
- {
- dd_localTopologyChecker(cr_->dd)->checkNumberOfBondedInteractions(localTopology_, x, box);
- }
if (flags & CGLO_STOPCM && !isInit)
{
process_and_stopcm_grp(fplog_, &vcm_, *mdAtoms_->mdatoms(), x, v);
}
}
-template<ComputeGlobalsAlgorithm algorithm>
-void ComputeGlobalsElement<algorithm>::setTopology(const gmx_localtop_t* top)
-{
- localTopology_ = top;
-}
-
template<ComputeGlobalsAlgorithm algorithm>
std::optional<SignallerCallback> ComputeGlobalsElement<algorithm>::registerEnergyCallback(EnergySignallerEvent event)
{
* \tparam algorithm The global reduction scheme
*/
template<ComputeGlobalsAlgorithm algorithm>
-class ComputeGlobalsElement final :
- public ISimulatorElement,
- public IEnergySignallerClient,
- public ITrajectorySignallerClient,
- public ITopologyHolderClient
+class ComputeGlobalsElement final : public ISimulatorElement, public IEnergySignallerClient, public ITrajectorySignallerClient
{
public:
//! Constructor
ObservablesReducer* observablesReducer);
private:
- //! ITopologyClient implementation
- void setTopology(const gmx_localtop_t* top) override;
//! IEnergySignallerClient implementation
std::optional<SignallerCallback> registerEnergyCallback(EnergySignallerEvent event) override;
//! ITrajectorySignallerClient implementation
StatePropagatorData* statePropagatorData_;
//! Pointer to the energy data (needed for the tensors and mu_tot)
EnergyData* energyData_;
- //! Pointer to the local topology (only needed for checkNumberOfBondedInteractions)
- const gmx_localtop_t* localTopology_;
//! Pointer to the free energy perturbation data
FreeEnergyPerturbationData* freeEnergyPerturbationData_;
{
return;
}
- std::unique_ptr<t_state> localState = statePropagatorData_->localState();
- t_state* globalState = statePropagatorData_->globalState();
+ t_state* localState = statePropagatorData_->localState();
+ t_state* globalState = statePropagatorData_->globalState();
// constant choices for this call to dd_partition_system
const bool verbose = isVerbose_ && (step % verbosePrintInterval_ == 0 || step == inputrec_->init_step);
}
if (isMasterState)
{
- dd_collect_state(cr_->dd, localState.get(), globalState);
+ dd_collect_state(cr_->dd, localState, globalState);
}
// Distribute the charge groups over the nodes from the master node
- partitionSystem(verbose, isMasterState, nstglobalcomm_, wcycle_, std::move(localState), globalState);
+ partitionSystem(verbose, isMasterState, nstglobalcomm_, wcycle_, localState, globalState);
}
-void DomDecHelper::partitionSystem(bool verbose,
- bool isMasterState,
- int nstglobalcomm,
- gmx_wallcycle* wcycle,
- std::unique_ptr<t_state> localState,
- t_state* globalState)
+void DomDecHelper::partitionSystem(bool verbose,
+ bool isMasterState,
+ int nstglobalcomm,
+ gmx_wallcycle* wcycle,
+ t_state* localState,
+ t_state* globalState)
{
ForceBuffers* forcePointer = statePropagatorData_->forcePointer();
*inputrec_,
imdSession_,
pull_work_,
- localState.get(),
+ localState,
forcePointer,
mdAtoms_,
- topologyHolder_->localTopology_.get(),
+ topologyHolder_->localTopology_,
fr_,
vsite_,
constr_,
nrnb_,
wcycle,
verbose);
- statePropagatorData_->setLocalState(std::move(localState));
+ statePropagatorData_->setLocalState(localState);
for (const auto& callback : domdecCallbacks_)
{
callback();
TopologyHolder* topologyHolder_;
//! Helper function unifying the DD partitioning calls in setup() and run()
- void partitionSystem(bool verbose,
- bool isMasterState,
- int nstglobalcomm,
- gmx_wallcycle* wcycle,
- std::unique_ptr<t_state> localState,
- t_state* globalState);
+ void partitionSystem(bool verbose,
+ bool isMasterState,
+ int nstglobalcomm,
+ gmx_wallcycle* wcycle,
+ t_state* localState,
+ t_state* globalState);
// Access to ISimulator data
//! Handles logging.
legacySimulatorData->fplog,
legacySimulatorData->cr,
legacySimulatorData->state_global,
+ legacySimulatorData->state,
legacySimulatorData->fr->nbv->useGpu(),
legacySimulatorData->fr->bMolPBC,
legacySimulatorData->mdrunOptions.writeConfout,
// Build topology holder
algorithm.topologyHolder_ = topologyHolderBuilder_.build(legacySimulatorData_->top_global,
+ legacySimulatorData_->top,
legacySimulatorData_->cr,
legacySimulatorData_->inputrec,
legacySimulatorData_->fr,
FILE* fplog,
const t_commrec* cr,
t_state* globalState,
+ t_state* localState,
bool useGPU,
bool canMoleculesBeDistributedOverPBC,
bool writeFinalConfiguration,
// Local state only becomes valid now.
if (DOMAINDECOMP(cr))
{
- auto localState = std::make_unique<t_state>();
- dd_init_local_state(*cr->dd, globalState, localState.get());
+ dd_init_local_state(*cr->dd, globalState, localState);
stateHasVelocities = ((localState->flags & enumValueToBitMask(StateEntry::V)) != 0);
- setLocalState(std::move(localState));
+ setLocalState(localState);
}
else
{
return totalNumAtoms_;
}
-std::unique_ptr<t_state> StatePropagatorData::localState()
+t_state* StatePropagatorData::localState()
{
- auto state = std::make_unique<t_state>();
- state->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V)
- | enumValueToBitMask(StateEntry::Box);
- state_change_natoms(state.get(), localNAtoms_);
- state->x = x_;
- state->v = v_;
- copy_mat(box_, state->box);
- state->ddp_count = ddpCount_;
- state->ddp_count_cg_gl = ddpCountCgGl_;
- state->cg_gl = cgGl_;
- return state;
+ localState_->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V)
+ | enumValueToBitMask(StateEntry::Box);
+ state_change_natoms(localState_, localNAtoms_);
+ std::swap(localState_->x, x_);
+ std::swap(localState_->v, v_);
+ copy_mat(box_, localState_->box);
+ localState_->ddp_count = ddpCount_;
+ localState_->ddp_count_cg_gl = ddpCountCgGl_;
+ localState_->cg_gl = cgGl_;
+ return localState_;
}
-void StatePropagatorData::setLocalState(std::unique_ptr<t_state> state)
+std::unique_ptr<t_state> StatePropagatorData::copyLocalState(std::unique_ptr<t_state> copy)
{
+ copy->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V)
+ | enumValueToBitMask(StateEntry::Box);
+ state_change_natoms(copy.get(), localNAtoms_);
+ copy->x = x_;
+ copy->v = v_;
+ copy_mat(box_, copy->box);
+ copy->ddp_count = ddpCount_;
+ copy->ddp_count_cg_gl = ddpCountCgGl_;
+ copy->cg_gl = cgGl_;
+ return copy;
+}
+
+void StatePropagatorData::setLocalState(t_state* state)
+{
+ localState_ = state;
localNAtoms_ = state->natoms;
- x_.resizeWithPadding(localNAtoms_);
previousX_.resizeWithPadding(localNAtoms_);
- v_.resizeWithPadding(localNAtoms_);
- x_ = state->x;
- v_ = state->v;
+ std::swap(x_, state->x);
+ std::swap(v_, state->v);
copy_mat(state->box, box_);
copyPosition();
ddpCount_ = state->ddp_count;
void StatePropagatorData::Element::saveState()
{
- GMX_ASSERT(!localStateBackup_, "Save state called again before previous state was written.");
- localStateBackup_ = statePropagatorData_->localState();
+ GMX_ASSERT(!localStateBackupValid_,
+ "Save state called again before previous state was written.");
+ localStateBackup_ = statePropagatorData_->copyLocalState(std::move(localStateBackup_));
if (freeEnergyPerturbationData_)
{
localStateBackup_->fep_state = freeEnergyPerturbationData_->currentFEPState();
localStateBackup_->flags |=
enumValueToBitMask(StateEntry::Lambda) | enumValueToBitMask(StateEntry::FepState);
}
+ localStateBackupValid_ = true;
}
std::optional<SignallerCallback> StatePropagatorData::Element::registerTrajectorySignallerCallback(TrajectoryEvent event)
wallcycle_stop(mdoutf_get_wcycle(outf), WallCycleCounter::Traj);
return;
}
- GMX_ASSERT(localStateBackup_, "Trajectory writing called, but no state saved.");
+ GMX_ASSERT(localStateBackupValid_, "Trajectory writing called, but no state saved.");
// TODO: This is only used for CPT - needs to be filled when we turn CPT back on
ObservablesHistory* observablesHistory = nullptr;
if (currentStep != lastStep_ || !isRegularSimulationEnd_)
{
- localStateBackup_.reset();
+ localStateBackupValid_ = false;
}
wallcycle_stop(mdoutf_get_wcycle(outf), WallCycleCounter::Traj);
}
return;
}
- GMX_ASSERT(localStateBackup_, "Final trajectory writing called, but no state saved.");
+ GMX_ASSERT(localStateBackupValid_, "Final trajectory writing called, but no state saved.");
wallcycle_start(mdoutf_get_wcycle(outf), WallCycleCounter::Traj);
if (DOMAINDECOMP(cr_))
nstvout_(nstvout),
nstfout_(nstfout),
nstxout_compressed_(nstxout_compressed),
+ localStateBackup_(std::make_unique<t_state>()),
writeOutStep_(-1),
freeEnergyPerturbationData_(nullptr),
isRegularSimulationEnd_(false),
FILE* fplog,
const t_commrec* cr,
t_state* globalState,
+ t_state* localState,
bool useGPU,
bool canMoleculesBeDistributedOverPBC,
bool writeFinalConfiguration,
void doCheckpointData(CheckpointData<operation>* checkpointData);
// Access to legacy state
- //! Get a deep copy of the current state in legacy format
- std::unique_ptr<t_state> localState();
- //! Update the current state with a state in legacy format
- void setLocalState(std::unique_ptr<t_state> state);
+ //! Give ownership of local state resources in legacy format
+ t_state* localState();
+ //! Take ownership of local state resources in legacy format
+ void setLocalState(t_state* state);
+ /*! \brief Deep copy the local state into the provided copy and
+ * return it
+ *
+ * In order to minimize reallocations, this function takes as a sink
+ * a local state object owned by the caller, copies the current local
+ * state into it, and returns the same object via a move.
+ */
+ std::unique_ptr<t_state> copyLocalState(std::unique_ptr<t_state> copy);
//! Get a pointer to the global state
t_state* globalState();
//! Get a force pointer
// Access to ISimulator data
//! Full simulation state (only non-nullptr on master rank).
t_state* globalState_;
+ //! Local simulation state
+ t_state* localState_;
};
/*! \internal
//! Pointer to keep a backup of the state for later writeout
std::unique_ptr<t_state> localStateBackup_;
+ /*! \brief Whether the contents of localStateBackup_ are logically valid
+ *
+ * This ensures that we don't make a second backup without consuming the
+ * first. */
+ bool localStateBackupValid_ = false;
//! Step at which next writeout occurs
Step writeOutStep_;
//! Backup current state
{
TopologyHolder::TopologyHolder(std::vector<ITopologyHolderClient*> clients,
const gmx_mtop_t& globalTopology,
+ gmx_localtop_t* localTopology,
const t_commrec* cr,
const t_inputrec* inputrec,
t_forcerec* fr,
MDAtoms* mdAtoms,
Constraints* constr,
VirtualSitesHandler* vsite) :
- globalTopology_(globalTopology),
- localTopology_(std::make_unique<gmx_localtop_t>(globalTopology.ffparams)),
- clients_(std::move(clients))
+ globalTopology_(globalTopology), localTopology_(localTopology), clients_(std::move(clients))
{
if (!DOMAINDECOMP(cr))
{
// TopologyHolder has no access to the forces, so we are passing a nullptr
// TODO: Find a unique approach to resizing the forces in modular simulator (#3461)
mdAlgorithmsSetupAtomData(
- cr, *inputrec, globalTopology, localTopology_.get(), fr, nullptr, mdAtoms, constr, vsite, nullptr);
+ cr, *inputrec, globalTopology, localTopology_, fr, nullptr, mdAtoms, constr, vsite, nullptr);
}
// Send copy of initial topology to clients
updateLocalTopology();
{
for (auto& client : clients_)
{
- client->setTopology(localTopology_.get());
+ client->setTopology(localTopology_);
}
}
DomDecCallback TopologyHolder::registerDomDecCallback()
//! Constructor
TopologyHolder(std::vector<ITopologyHolderClient*> clients,
const gmx_mtop_t& globalTopology,
+ gmx_localtop_t* localTopology,
const t_commrec* cr,
const t_inputrec* inputrec,
t_forcerec* fr,
//! Constant reference to the global topology
const gmx_mtop_t& globalTopology_;
//! Pointer to the currently valid local topology
- std::unique_ptr<gmx_localtop_t> localTopology_;
+ gmx_localtop_t* localTopology_;
//! List of clients to be updated if local topology changes
std::vector<ITopologyHolderClient*> clients_;
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff