From: Mark Abraham Date: Thu, 23 Sep 2021 09:57:16 +0000 (+0000) Subject: Use ObservablesReducer for check of DD bonded interaction count. X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=4cec4672756534c7c5c22729d29c30c38de2c791;p=alexxy%2Fgromacs.git Use ObservablesReducer for check of DD bonded interaction count. The lifetime of the local and global state has been shifted, so that the LocalTopologyChecker can now be initialized with the handles it might need to use to report an error, even though one of those handles will not contain valid data until after the first DD partition. This simplifies ComputeGlobalsElement from the modular simulator. The changes to StatePropagatorData motivated by this change reduce the number of copies of x and v vectors that were being done every partitioning step. Those are needed only when the state backup is taken so that trajectory output can be handled correctly. The check is now implemented for minimizers, where previously it was not. Refs #3887 #3421 --- diff --git a/src/gromacs/domdec/builder.h b/src/gromacs/domdec/builder.h index e5b9a8d92b..6bebfc33b5 100644 --- a/src/gromacs/domdec/builder.h +++ b/src/gromacs/domdec/builder.h @@ -53,8 +53,10 @@ struct gmx_domdec_t; struct gmx_mtop_t; +struct gmx_localtop_t; struct t_commrec; struct t_inputrec; +class t_state; namespace gmx { @@ -64,6 +66,7 @@ class RangePartitioning; struct DomdecOptions; struct MdrunOptions; struct MDModulesNotifiers; +class ObservablesReducerBuilder; template class ArrayRef; @@ -96,7 +99,10 @@ public: //! Destructor ~DomainDecompositionBuilder(); //! Build the resulting DD manager - gmx_domdec_t* build(LocalAtomSetManager* atomSets); + gmx_domdec_t* build(LocalAtomSetManager* atomSets, + const gmx_localtop_t& localTopology, + const t_state& localState, + ObservablesReducerBuilder* observablesReducerBuilder); private: class Impl; diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index 5f26c40848..f4ddcc70ee 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -2860,7 +2860,10 @@ public: bool directGpuCommUsedWithGpuUpdate); //! Build the resulting DD manager - gmx_domdec_t* build(LocalAtomSetManager* atomSets); + gmx_domdec_t* build(LocalAtomSetManager* atomSets, + const gmx_localtop_t& localTopology, + const t_state& localState, + ObservablesReducerBuilder* observablesReducerBuilder); //! Objects used in constructing and configuring DD //! { @@ -2996,7 +2999,10 @@ DomainDecompositionBuilder::Impl::Impl(const MDLogger& mdlog, mdlog_, ddSettings_, options_.rankOrder, ddRankSetup_, cr_, ddCellIndex_, &pmeRanks_); } -gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets) +gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets, + const gmx_localtop_t& localTopology, + const t_state& localState, + ObservablesReducerBuilder* observablesReducerBuilder) { gmx_domdec_t* dd = new gmx_domdec_t(ir_); @@ -3034,7 +3040,7 @@ gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomS dd->atomSets = atomSets; dd->localTopologyChecker = std::make_unique( - mdlog_, cr_, mtop_, dd->comm->systemInfo.useUpdateGroups); + mdlog_, cr_, mtop_, localTopology, localState, dd->comm->systemInfo.useUpdateGroups, observablesReducerBuilder); return dd; } @@ -3072,9 +3078,12 @@ DomainDecompositionBuilder::DomainDecompositionBuilder(const MDLogger& { } -gmx_domdec_t* DomainDecompositionBuilder::build(LocalAtomSetManager* atomSets) +gmx_domdec_t* DomainDecompositionBuilder::build(LocalAtomSetManager* atomSets, + const gmx_localtop_t& localTopology, + const t_state& localState, + ObservablesReducerBuilder* observablesReducerBuilder) { - return impl_->build(atomSets); + return impl_->build(atomSets, localTopology, localState, observablesReducerBuilder); } DomainDecompositionBuilder::~DomainDecompositionBuilder() = default; @@ -3233,16 +3242,6 @@ void communicateGpuHaloForces(const t_commrec& cr, bool accumulateForces) } } -const gmx::LocalTopologyChecker& dd_localTopologyChecker(const gmx_domdec_t& dd) -{ - return *dd.localTopologyChecker; -} - -gmx::LocalTopologyChecker* dd_localTopologyChecker(gmx_domdec_t* dd) -{ - return dd->localTopologyChecker.get(); -} - void dd_init_local_state(const gmx_domdec_t& dd, const t_state* state_global, t_state* state_local) { std::array buf; diff --git a/src/gromacs/domdec/domdec.h b/src/gromacs/domdec/domdec.h index ab5363ce3e..bb964de98e 100644 --- a/src/gromacs/domdec/domdec.h +++ b/src/gromacs/domdec/domdec.h @@ -87,7 +87,6 @@ namespace gmx struct AtomInfoWithinMoleculeBlock; class DeviceStreamManager; class ForceWithShiftForces; -class LocalTopologyChecker; class MDLogger; class RangePartitioning; class VirtualSitesHandler; @@ -239,16 +238,6 @@ void dd_move_x_and_v_vsites(const gmx_domdec_t& dd, const matrix box, rvec* x, r */ gmx::ArrayRef dd_constraints_nlocalatoms(const gmx_domdec_t* dd); -/*! \brief Const getter for the local topology checker - * - * \returns Const handle to local topology checker */ -const gmx::LocalTopologyChecker& dd_localTopologyChecker(const gmx_domdec_t& dd); - -/*! \brief Getter for the local topology checker - * - * \returns Handle to local topology checker */ -gmx::LocalTopologyChecker* dd_localTopologyChecker(gmx_domdec_t* dd); - /*! \brief Construct local state */ void dd_init_local_state(const gmx_domdec_t& dd, const t_state* state_global, t_state* local_state); diff --git a/src/gromacs/domdec/localtopologychecker.cpp b/src/gromacs/domdec/localtopologychecker.cpp index c4af1d1d32..8db5e261be 100644 --- a/src/gromacs/domdec/localtopologychecker.cpp +++ b/src/gromacs/domdec/localtopologychecker.cpp @@ -54,6 +54,8 @@ #include "gromacs/domdec/reversetopology.h" #include "gromacs/gmxlib/network.h" #include "gromacs/mdtypes/commrec.h" +#include "gromacs/mdtypes/observablesreducer.h" +#include "gromacs/mdtypes/state.h" #include "gromacs/topology/idef.h" #include "gromacs/topology/ifunc.h" #include "gromacs/topology/mtop_util.h" @@ -257,7 +259,7 @@ static void printMissingInteractionsAtoms(const MDLogger& mdlog, const int numBondedInteractionsOverAllDomains, const int expectedNumGlobalBondedInteractions, const gmx_mtop_t& top_global, - const gmx_localtop_t* top_local, + const gmx_localtop_t& top_local, ArrayRef x, const matrix box) { @@ -274,7 +276,7 @@ static void printMissingInteractionsAtoms(const MDLogger& mdlog, for (int ftype = 0; ftype < F_NRE; ftype++) { const int nral = NRAL(ftype); - cl[ftype] = top_local->idef.il[ftype].size() / (1 + nral); + cl[ftype] = top_local.idef.il[ftype].size() / (1 + nral); } gmx_sumi(F_NRE, cl, cr); @@ -318,7 +320,7 @@ static void printMissingInteractionsAtoms(const MDLogger& mdlog, } } - printMissingInteractionsAtoms(mdlog, cr, top_global, top_local->idef); + printMissingInteractionsAtoms(mdlog, cr, top_global, top_local.idef); write_dd_pdb("dd_dump_err", 0, "dump", top_global, cr, -1, as_rvec_array(x.data()), box); std::string errorMessage; @@ -357,8 +359,12 @@ class LocalTopologyChecker::Impl { public: //! Constructor - Impl(const MDLogger& mdlog, const t_commrec* cr, const gmx_mtop_t& mtop, bool useUpdateGroups); - + Impl(const MDLogger& mdlog, + const t_commrec* cr, + const gmx_mtop_t& mtop, + const gmx_localtop_t& localTopology, + const t_state& localState, + bool useUpdateGroups); //! Objects used when reporting that interactions are missing //! { //! Logger @@ -367,25 +373,25 @@ public: const t_commrec* cr_; //! Global system topology const gmx_mtop_t& mtop_; + //! Local topology + const gmx_localtop_t& localTopology_; + //! Local state + const t_state& localState_; //! } - /*! \brief Number of bonded interactions found in the local - * topology for this domain. */ - int numBondedInteractionsToReduce_ = 0; - /*! \brief Whether to check at the next global communication - * stage the total number of bonded interactions found. - * - * Cleared after that number is found. */ - bool shouldCheckNumberOfBondedInteractions_ = false; - /*! \brief The total number of bonded interactions found in - * the local topology across all domains. + /*! \brief View used for computing the global number of bonded interactions. * - * Only has a value after reduction across all ranks, which is - * removed when it is again time to check after a new - * partition. */ - std::optional numBondedInteractionsOverAllDomains_; - //! The number of bonded interactions computed from the full system topology - int expectedNumGlobalBondedInteractions_ = 0; + * Can be written any time, but that is only useful when followed + * by a call of the callbackToRequireReduction. Useful to read + * only from the callback that the ObservablesReducer will later + * make after reduction. */ + gmx::ArrayRef reductionBuffer_; + /*! \brief Callback used after repartitioning to require reduction + * of numBondedInteractionsToReduce so that the total number of + * bonded interactions can be checked. */ + gmx::ObservablesReducerBuilder::CallbackToRequireReduction callbackToRequireReduction_; + /*! \brief The expected number of global bonded interactions from the system topology */ + int expectedNumGlobalBondedInteractions_; }; @@ -409,23 +415,57 @@ static int computeExpectedNumGlobalBondedInteractions(const gmx_mtop_t& mtop, co return expectedNumGlobalBondedInteractions; } -LocalTopologyChecker::Impl::Impl(const MDLogger& mdlog, - const t_commrec* cr, - const gmx_mtop_t& mtop, - const bool useUpdateGroups) : +LocalTopologyChecker::Impl::Impl(const MDLogger& mdlog, + const t_commrec* cr, + const gmx_mtop_t& mtop, + const gmx_localtop_t& localTopology, + const t_state& localState, + bool useUpdateGroups) : mdlog_(mdlog), cr_(cr), mtop_(mtop), + localTopology_(localTopology), + localState_(localState), expectedNumGlobalBondedInteractions_(computeExpectedNumGlobalBondedInteractions(mtop, useUpdateGroups)) { } -LocalTopologyChecker::LocalTopologyChecker(const MDLogger& mdlog, - const t_commrec* cr, - const gmx_mtop_t& mtop, - const bool useUpdateGroups) : - impl_(std::make_unique(mdlog, cr, mtop, useUpdateGroups)) +LocalTopologyChecker::LocalTopologyChecker(const MDLogger& mdlog, + const t_commrec* cr, + const gmx_mtop_t& mtop, + const gmx_localtop_t& localTopology, + const t_state& localState, + const bool useUpdateGroups, + ObservablesReducerBuilder* observablesReducerBuilder) : + impl_(std::make_unique(mdlog, cr, mtop, localTopology, localState, useUpdateGroups)) { + Impl* impl = impl_.get(); + ObservablesReducerBuilder::CallbackFromBuilder callbackFromBuilder = + [impl](ObservablesReducerBuilder::CallbackToRequireReduction c, gmx::ArrayRef v) { + impl->callbackToRequireReduction_ = std::move(c); + impl->reductionBuffer_ = v; + }; + + // Make the callback that runs afer reduction. + ObservablesReducerBuilder::CallbackAfterReduction callbackAfterReduction = [impl](gmx::Step /*step*/) { + // Get the total after reduction + int numTotalBondedInteractionsFound = impl->reductionBuffer_[0]; + if (numTotalBondedInteractionsFound != impl->expectedNumGlobalBondedInteractions_) + { + // Give error and exit + dd_print_missing_interactions(impl->mdlog_, + impl->cr_, + numTotalBondedInteractionsFound, + impl->expectedNumGlobalBondedInteractions_, + impl->mtop_, + impl->localTopology_, + impl->localState_.x, + impl->localState_.box); // Does not return + } + }; + + observablesReducerBuilder->addSubscriber( + 1, std::move(callbackFromBuilder), std::move(callbackAfterReduction)); } LocalTopologyChecker::~LocalTopologyChecker() = default; @@ -440,59 +480,25 @@ LocalTopologyChecker& LocalTopologyChecker::operator=(LocalTopologyChecker&& oth void LocalTopologyChecker::scheduleCheckOfLocalTopology(const int numBondedInteractionsToReduce) { - impl_->numBondedInteractionsToReduce_ = numBondedInteractionsToReduce; - // Note that it's possible for this to still be true from the last - // time it was set, e.g. if repartitioning was triggered before - // global communication that would have acted on the true - // value. This could happen for example when replica exchange took - // place soon after a partition. - impl_->shouldCheckNumberOfBondedInteractions_ = true; - // Clear the old global value, which is now invalid - impl_->numBondedInteractionsOverAllDomains_.reset(); -} - -bool LocalTopologyChecker::shouldCheckNumberOfBondedInteractions() const -{ - return impl_->shouldCheckNumberOfBondedInteractions_; -} - -int LocalTopologyChecker::numBondedInteractions() const -{ - return impl_->numBondedInteractionsToReduce_; -} - -void LocalTopologyChecker::setNumberOfBondedInteractionsOverAllDomains(const int newValue) -{ - GMX_RELEASE_ASSERT(!impl_->numBondedInteractionsOverAllDomains_.has_value(), - "Cannot set number of bonded interactions because it is already set"); - impl_->numBondedInteractionsOverAllDomains_.emplace(newValue); -} - -void LocalTopologyChecker::checkNumberOfBondedInteractions(const gmx_localtop_t* top_local, - ArrayRef x, - const matrix box) -{ - if (impl_->shouldCheckNumberOfBondedInteractions_) - { - GMX_RELEASE_ASSERT(impl_->numBondedInteractionsOverAllDomains_.has_value(), - "The check for the total number of bonded interactions requires the " - "value to have been reduced across all domains"); - if (impl_->numBondedInteractionsOverAllDomains_.value() != impl_->expectedNumGlobalBondedInteractions_) - { - dd_print_missing_interactions(impl_->mdlog_, - impl_->cr_, - impl_->numBondedInteractionsOverAllDomains_.value(), - impl_->expectedNumGlobalBondedInteractions_, - impl_->mtop_, - top_local, - x, - box); // Does not return - } - // Now that the value is set and the check complete, future - // global communication should not compute the value until - // after the next partitioning. - impl_->shouldCheckNumberOfBondedInteractions_ = false; - } + // Fill the reduction buffer with the value from this domain to reduce + impl_->reductionBuffer_[0] = double(numBondedInteractionsToReduce); + + // Pass the post-reduction callback to the ObservablesReducer via + // the callback it gave us for the purpose. + // + // Note that it's possible that the callbackAfterReduction is already + // outstanding, e.g. if repartitioning was triggered before + // observables were reduced. This could happen for example when + // replica exchange took place soon after a partition. If so, the + // callback will be called again. So long as there is no race + // between the calls to this function and the calls to + // ObservablesReducer for reduction, this will work correctly. It + // could be made safer e.g. with checks against duplicate + // callbacks, but there is no problem to solve. + // + // There is no need to check the return value from this callback, + // as it is not an error to request reduction at a future step. + impl_->callbackToRequireReduction_(ReductionRequirement::Eventually); } } // namespace gmx diff --git a/src/gromacs/domdec/localtopologychecker.h b/src/gromacs/domdec/localtopologychecker.h index 49766aa35e..253879021b 100644 --- a/src/gromacs/domdec/localtopologychecker.h +++ b/src/gromacs/domdec/localtopologychecker.h @@ -52,28 +52,31 @@ struct gmx_localtop_t; struct gmx_mtop_t; struct t_commrec; struct t_inputrec; +class t_state; namespace gmx { class MDLogger; -template -class ArrayRef; +class ObservablesReducerBuilder; } // namespace gmx namespace gmx { -/*! \brief Has responsibility for checking that the local topology - * distributed across domains describes a total number of bonded - * interactions that matches the system topology +/*! \libinternal + * \brief Has responsibility for checking that the local topology distributed + * across domains describes a total number of bonded interactions that matches + * the system topology * - * Because this check is not urgent, the communication that it - * requires is done at the next opportunity, rather than requiring - * extra communication. If the check fails, a fatal error stops - * execution. In principle, if there was a bug, GROMACS might crash in - * the meantime because of the wrong forces. However as a bug is - * unlikely, we optimize by avoiding creating extra overhead from - * communication. + * This uses the ObservablesReducer framework to check that the count + * of bonded interactions in the local topology made for each domain + * sums to the expected value. Because this check is not urgent, the + * communication that it requires is done at the next opportunity, + * rather than requiring extra communication. If the check fails, a + * fatal error stops execution. In principle, if there was a bug, + * GROMACS might crash in the meantime because of the wrong + * forces. However as a bug is unlikely we optimize by avoiding + * creating extra overhead from communication. */ class LocalTopologyChecker { @@ -82,9 +85,18 @@ public: * \param[in] mdlog Logger * \param[in] cr Communication object * \param[in] mtop Global system topology + * \param[in] localTopology The local topology + * \param[in] localState The local state * \param[in] useUpdateGroups Whether update groups are in use + * \param[in] observablesReducerBuilder Handle to builder for ObservablesReducer */ - LocalTopologyChecker(const MDLogger& mdlog, const t_commrec* cr, const gmx_mtop_t& mtop, bool useUpdateGroups); + LocalTopologyChecker(const MDLogger& mdlog, + const t_commrec* cr, + const gmx_mtop_t& mtop, + const gmx_localtop_t& localTopology, + const t_state& localState, + bool useUpdateGroups, + ObservablesReducerBuilder* observablesReducerBuilder); //! Destructor ~LocalTopologyChecker(); //! Move constructor @@ -97,27 +109,6 @@ public: * another module. */ void scheduleCheckOfLocalTopology(int numBondedInteractionsToReduce); - /*! \brief Return whether the total bonded interaction count across - * domains should be checked in observables reduction this step. */ - bool shouldCheckNumberOfBondedInteractions() const; - - //! Return the number of bonded interactions in this domain. - int numBondedInteractions() const; - - /*! \brief Set total bonded interaction count across domains. */ - void setNumberOfBondedInteractionsOverAllDomains(int newValue); - - /*! \brief Check whether bonded interactions are missing from the reverse topology - * produced by domain decomposition. - * - * \param[in] top_local Local topology for the error message - * \param[in] x Position vector for the error message - * \param[in] box Box matrix for the error message - */ - void checkNumberOfBondedInteractions(const gmx_localtop_t* top_local, - ArrayRef x, - const matrix box); - private: class Impl; std::unique_ptr impl_; diff --git a/src/gromacs/mdlib/md_support.cpp b/src/gromacs/mdlib/md_support.cpp index 297a41bf5a..1d25bc85c5 100644 --- a/src/gromacs/mdlib/md_support.cpp +++ b/src/gromacs/mdlib/md_support.cpp @@ -309,19 +309,17 @@ void compute_globals(gmx_global_stat* gstat, { gmx_bool bEner, bPres, bTemp; gmx_bool bStopCM, bGStat, bReadEkin, bEkinAveVel, bScaleEkin, bConstrain; - gmx_bool bCheckNumberOfBondedInteractions; real dvdl_ekin; /* translate CGLO flags to gmx_booleans */ - bStopCM = ((flags & CGLO_STOPCM) != 0); - bGStat = ((flags & CGLO_GSTAT) != 0); - bReadEkin = ((flags & CGLO_READEKIN) != 0); - bScaleEkin = ((flags & CGLO_SCALEEKIN) != 0); - bEner = ((flags & CGLO_ENERGY) != 0); - bTemp = ((flags & CGLO_TEMPERATURE) != 0); - bPres = ((flags & CGLO_PRESSURE) != 0); - bConstrain = ((flags & CGLO_CONSTRAINT) != 0); - bCheckNumberOfBondedInteractions = ((flags & CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS) != 0); + bStopCM = ((flags & CGLO_STOPCM) != 0); + bGStat = ((flags & CGLO_GSTAT) != 0); + bReadEkin = ((flags & CGLO_READEKIN) != 0); + bScaleEkin = ((flags & CGLO_SCALEEKIN) != 0); + bEner = ((flags & CGLO_ENERGY) != 0); + bTemp = ((flags & CGLO_TEMPERATURE) != 0); + bPres = ((flags & CGLO_PRESSURE) != 0); + bConstrain = ((flags & CGLO_CONSTRAINT) != 0); /* we calculate a full state kinetic energy either with full-step velocity verlet or half step where we need the pressure */ @@ -348,7 +346,7 @@ void compute_globals(gmx_global_stat* gstat, calc_vcm_grp(*mdatoms, x, v, vcm); } - if (bTemp || bStopCM || bPres || bEner || bConstrain || bCheckNumberOfBondedInteractions + if (bTemp || bStopCM || bPres || bEner || bConstrain || !observablesReducer->communicationBuffer().empty()) { if (!bGStat) diff --git a/src/gromacs/mdlib/md_support.h b/src/gromacs/mdlib/md_support.h index e43260a5cf..f257e4df3b 100644 --- a/src/gromacs/mdlib/md_support.h +++ b/src/gromacs/mdlib/md_support.h @@ -88,10 +88,6 @@ class SimulationSignaller; #define CGLO_READEKIN (1u << 10u) /* we need to reset the ekin rescaling factor here */ #define CGLO_SCALEEKIN (1u << 11u) -/* After a new DD partitioning, we need to set a flag to schedule - * global reduction of the total number of bonded interactions that - * will be computed, to check none are missing. */ -#define CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS (1u << 12u) /*! \brief Return the number of steps that will take place between * intra-simulation communications, given the constraints of the diff --git a/src/gromacs/mdlib/stat.cpp b/src/gromacs/mdlib/stat.cpp index 6be1c9d2cf..3f10a0ab3a 100644 --- a/src/gromacs/mdlib/stat.cpp +++ b/src/gromacs/mdlib/stat.cpp @@ -44,7 +44,6 @@ #include "gromacs/domdec/domdec.h" #include "gromacs/domdec/domdec_struct.h" -#include "gromacs/domdec/localtopologychecker.h" #include "gromacs/fileio/checkpoint.h" #include "gromacs/fileio/xtcio.h" #include "gromacs/gmxlib/network.h" @@ -159,19 +158,18 @@ void global_stat(const gmx_global_stat& gs, /* instead of current system, gmx_booleans for summing virial, kinetic energy, and other terms */ { int ie = 0, ifv = 0, isv = 0, irmsd = 0; - int idedl = 0, idedlo = 0, idvdll = 0, idvdlnl = 0, iepl = 0, icm = 0, imass = 0, ica = 0, inb = 0; + int idedl = 0, idedlo = 0, idvdll = 0, idvdlnl = 0, iepl = 0, icm = 0, imass = 0, ica = 0; int isig = -1; int icj = -1, ici = -1, icx = -1; - bool checkNumberOfBondedInteractions = (flags & CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS) != 0; - bool bVV = EI_VV(inputrec.eI); - bool bTemp = ((flags & CGLO_TEMPERATURE) != 0); - bool bEner = ((flags & CGLO_ENERGY) != 0); - bool bPres = ((flags & CGLO_PRESSURE) != 0); - bool bConstrVir = ((flags & CGLO_CONSTRAINT) != 0); - bool bEkinAveVel = (inputrec.eI == IntegrationAlgorithm::VV + bool bVV = EI_VV(inputrec.eI); + bool bTemp = ((flags & CGLO_TEMPERATURE) != 0); + bool bEner = ((flags & CGLO_ENERGY) != 0); + bool bPres = ((flags & CGLO_PRESSURE) != 0); + bool bConstrVir = ((flags & CGLO_CONSTRAINT) != 0); + bool bEkinAveVel = (inputrec.eI == IntegrationAlgorithm::VV || (inputrec.eI == IntegrationAlgorithm::VVAK && bPres)); - bool bReadEkin = ((flags & CGLO_READEKIN) != 0); + bool bReadEkin = ((flags & CGLO_READEKIN) != 0); // This structure implements something akin to a vector. As // modules add their data into it with add_bin[rd], they save the @@ -279,15 +277,6 @@ void global_stat(const gmx_global_stat& gs, } } - double nb; - if (checkNumberOfBondedInteractions) - { - GMX_RELEASE_ASSERT(DOMAINDECOMP(cr), - "No need to check number of bonded interactions when not using domain " - "decomposition"); - nb = cr->dd->localTopologyChecker->numBondedInteractions(); - inb = add_bind(rb, 1, &nb); - } if (!sig.empty()) { isig = add_binr(rb, sig); @@ -385,15 +374,6 @@ void global_stat(const gmx_global_stat& gs, } } - if (checkNumberOfBondedInteractions) - { - extract_bind(rb, inb, 1, &nb); - GMX_RELEASE_ASSERT(DOMAINDECOMP(cr), - "No need to check number of bonded interactions when not using domain " - "decomposition"); - cr->dd->localTopologyChecker->setNumberOfBondedInteractionsOverAllDomains(gmx::roundToInt(nb)); - } - if (!sig.empty()) { extract_binr(rb, isig, sig); diff --git a/src/gromacs/mdlib/update_vv.cpp b/src/gromacs/mdlib/update_vv.cpp index 279e77c895..8f8709e2f8 100644 --- a/src/gromacs/mdlib/update_vv.cpp +++ b/src/gromacs/mdlib/update_vv.cpp @@ -43,7 +43,6 @@ #include -#include "gromacs/domdec/domdec.h" #include "gromacs/domdec/localtopologychecker.h" #include "gromacs/gmxlib/nrnb.h" #include "gromacs/math/vec.h" @@ -82,7 +81,6 @@ void integrateVVFirstStep(int64_t step, t_fcdata* fcdata, t_extmass* MassQ, t_vcm* vcm, - const gmx_localtop_t& top, gmx_enerdata_t* enerd, gmx::ObservablesReducer* observablesReducer, gmx_ekindata_t* ekind, @@ -191,10 +189,6 @@ void integrateVVFirstStep(int64_t step, ((bGStat ? CGLO_GSTAT : 0) | (bCalcEner ? CGLO_ENERGY : 0) | (bTemp ? CGLO_TEMPERATURE : 0) | (bPres ? CGLO_PRESSURE : 0) | (bPres ? CGLO_CONSTRAINT : 0) | (bStopCM ? CGLO_STOPCM : 0) | CGLO_SCALEEKIN); - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat, cr, ir, @@ -226,11 +220,6 @@ void integrateVVFirstStep(int64_t step, time step kinetic energy for the pressure (always true now, since we want accurate statistics). b) If we are using EkinAveEkin for the kinetic energy for the temperature control, we still feed in EkinAveVel because it's needed for the pressure */ - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } if (bStopCM) { process_and_stopcm_grp( diff --git a/src/gromacs/mdlib/update_vv.h b/src/gromacs/mdlib/update_vv.h index 9cc7bdb391..2ff6f79f7f 100644 --- a/src/gromacs/mdlib/update_vv.h +++ b/src/gromacs/mdlib/update_vv.h @@ -50,8 +50,6 @@ class gmx_ekindata_t; struct gmx_enerdata_t; struct gmx_global_stat; -struct gmx_localtop_t; -struct gmx_mtop_t; struct gmx_wallcycle; struct pull_t; struct t_commrec; @@ -68,7 +66,6 @@ namespace gmx { class Constraints; class ForceBuffers; -class MDLogger; class ObservablesReducer; class SimulationSignaller; class Update; @@ -90,7 +87,6 @@ enum class StartingBehavior : int; * \param[in] fcdata Force calculation data. * \param[in] MassQ Mass/pressure data. * \param[in] vcm Center of mass motion removal. - * \param[in] top Local topology. * \param[in] enerd Energy data. * \param[in] observablesReducer Pointer to the \c ObservablesReducer object * \param[in] ekind Kinetic energy data. @@ -133,7 +129,6 @@ void integrateVVFirstStep(int64_t step, t_fcdata* fcdata, t_extmass* MassQ, t_vcm* vcm, - const gmx_localtop_t& top, gmx_enerdata_t* enerd, gmx::ObservablesReducer* observablesReducer, gmx_ekindata_t* ekind, diff --git a/src/gromacs/mdrun/isimulator.h b/src/gromacs/mdrun/isimulator.h index e7cf49b830..fcddbeeacc 100644 --- a/src/gromacs/mdrun/isimulator.h +++ b/src/gromacs/mdrun/isimulator.h @@ -47,6 +47,7 @@ class energyhistory_t; class gmx_ekindata_t; struct gmx_enerdata_t; struct gmx_enfrot; +struct gmx_localtop_t; struct gmx_mtop_t; struct gmx_membed_t; struct gmx_multisim_t; @@ -130,7 +131,9 @@ public: pull_t* pull_work, t_swap* swap, const gmx_mtop_t& top_global, + gmx_localtop_t* top, t_state* state_global, + t_state* state, ObservablesHistory* observablesHistory, MDAtoms* mdAtoms, t_nrnb* nrnb, @@ -165,7 +168,9 @@ public: pull_work(pull_work), swap(swap), top_global(top_global), + top(top), state_global(state_global), + state(state), observablesHistory(observablesHistory), mdAtoms(mdAtoms), nrnb(nrnb), @@ -223,8 +228,12 @@ public: t_swap* swap; //! Full system topology. const gmx_mtop_t& top_global; + //! Handle to local simulation topology. + gmx_localtop_t* top; //! Full simulation state (only non-nullptr on master rank). t_state* state_global; + //! Handle to local state of the simulation. + t_state* state; //! History of simulation observables. ObservablesHistory* observablesHistory; //! Atom parameters for this domain. diff --git a/src/gromacs/mdrun/md.cpp b/src/gromacs/mdrun/md.cpp index 58d1da65fe..fdb28998e2 100644 --- a/src/gromacs/mdrun/md.cpp +++ b/src/gromacs/mdrun/md.cpp @@ -357,11 +357,6 @@ void gmx::LegacySimulator::do_md() } } - // Local state only becomes valid now. - std::unique_ptr stateInstance; - t_state* state; - - gmx_localtop_t top(top_global.ffparams); ObservablesReducer observablesReducer = observablesReducerBuilder->build(); ForceBuffers f(simulationWork.useMts, @@ -371,8 +366,7 @@ void gmx::LegacySimulator::do_md() const t_mdatoms* md = mdAtoms->mdatoms(); if (DOMAINDECOMP(cr)) { - stateInstance = std::make_unique(); - state = stateInstance.get(); + // Local state only becomes valid now. dd_init_local_state(*cr->dd, state_global, state); /* Distribute the charge groups over the nodes from the master node */ @@ -390,7 +384,7 @@ void gmx::LegacySimulator::do_md() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -407,11 +401,9 @@ void gmx::LegacySimulator::do_md() else { state_change_natoms(state_global, state_global->natoms); - /* Copy the pointer to the global state */ - state = state_global; /* Generate and initialize new topology */ - mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc); + mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc); upd.updateAfterPartition(state->natoms, md->cFREEZE ? gmx::arrayRefFromArray(md->cFREEZE, md->nr) @@ -652,11 +644,6 @@ void gmx::LegacySimulator::do_md() cglo_flags_iteration |= CGLO_STOPCM; cglo_flags_iteration &= ~CGLO_TEMPERATURE; } - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions() - && cgloIteration == 0) - { - cglo_flags_iteration |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat, cr, ir, @@ -693,11 +680,6 @@ void gmx::LegacySimulator::do_md() inc_nrnb(nrnb, eNR_STOPCM, md->homenr); } } - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } if (ir->eI == IntegrationAlgorithm::VVAK) { /* a second call to get the half step temperature initialized as well */ @@ -1006,7 +988,7 @@ void gmx::LegacySimulator::do_md() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -1048,10 +1030,6 @@ void gmx::LegacySimulator::do_md() * the full step kinetic energy and possibly for T-coupling.*/ /* This may not be quite working correctly yet . . . . */ int cglo_flags = CGLO_GSTAT | CGLO_TEMPERATURE; - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat, cr, ir, @@ -1076,11 +1054,6 @@ void gmx::LegacySimulator::do_md() cglo_flags, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } } clear_mat(force_vir); @@ -1143,7 +1116,7 @@ void gmx::LegacySimulator::do_md() pull_work, bNS, force_flags, - &top, + top, constr, enerd, state->natoms, @@ -1197,7 +1170,7 @@ void gmx::LegacySimulator::do_md() step, nrnb, wcycle, - &top, + top, state->box, state->x.arrayRefWithPadding(), &state->hist, @@ -1236,7 +1209,6 @@ void gmx::LegacySimulator::do_md() &fcdata, &MassQ, &vcm, - top, enerd, &observablesReducer, ekind, @@ -1520,7 +1492,7 @@ void gmx::LegacySimulator::do_md() integrator->set(stateGpu->getCoordinates(), stateGpu->getVelocities(), stateGpu->getForces(), - top.idef, + top->idef, *md); // Copy data to the GPU after buffers might have being reinitialized @@ -1714,17 +1686,9 @@ void gmx::LegacySimulator::do_md() (bGStat ? CGLO_GSTAT : 0) | (!EI_VV(ir->eI) && bCalcEner ? CGLO_ENERGY : 0) | (!EI_VV(ir->eI) && bStopCM ? CGLO_STOPCM : 0) | (!EI_VV(ir->eI) ? CGLO_TEMPERATURE : 0) - | (!EI_VV(ir->eI) ? CGLO_PRESSURE : 0) | CGLO_CONSTRAINT - | (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions() - ? CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS - : 0), + | (!EI_VV(ir->eI) ? CGLO_PRESSURE : 0) | CGLO_CONSTRAINT, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } if (!EI_VV(ir->eI) && bStopCM) { process_and_stopcm_grp( @@ -1982,7 +1946,7 @@ void gmx::LegacySimulator::do_md() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, diff --git a/src/gromacs/mdrun/mimic.cpp b/src/gromacs/mdrun/mimic.cpp index 2c08bdf15f..82852d21bf 100644 --- a/src/gromacs/mdrun/mimic.cpp +++ b/src/gromacs/mdrun/mimic.cpp @@ -284,16 +284,9 @@ void gmx::LegacySimulator::do_mimic() } } - // Local state only becomes valid now. - std::unique_ptr stateInstance; - t_state* state; - - gmx_localtop_t top(top_global.ffparams); - if (DOMAINDECOMP(cr)) { - stateInstance = std::make_unique(); - state = stateInstance.get(); + // Local state only becomes valid now. dd_init_local_state(*cr->dd, state_global, state); /* Distribute the charge groups over the nodes from the master node */ @@ -311,7 +304,7 @@ void gmx::LegacySimulator::do_mimic() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -322,10 +315,7 @@ void gmx::LegacySimulator::do_mimic() else { state_change_natoms(state_global, state_global->natoms); - /* Copy the pointer to the global state */ - state = state_global; - - mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc); + mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc); } auto* mdatoms = mdAtoms->mdatoms(); @@ -347,11 +337,7 @@ void gmx::LegacySimulator::do_mimic() int64_t step_rel = 0; { - int cglo_flags = CGLO_GSTAT; - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } + int cglo_flags = CGLO_GSTAT; bool bSumEkinhOld = false; t_vcm* vcm = nullptr; compute_globals(gstat, @@ -378,11 +364,6 @@ void gmx::LegacySimulator::do_mimic() cglo_flags, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } } if (MASTER(cr)) @@ -497,7 +478,7 @@ void gmx::LegacySimulator::do_mimic() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -536,7 +517,7 @@ void gmx::LegacySimulator::do_mimic() pull_work, bNS, force_flags, - &top, + top, constr, enerd, state->natoms, @@ -579,7 +560,7 @@ void gmx::LegacySimulator::do_mimic() step, nrnb, wcycle, - &top, + top, state->box, state->x.arrayRefWithPadding(), &state->hist, @@ -640,10 +621,6 @@ void gmx::LegacySimulator::do_mimic() SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal); int cglo_flags = CGLO_GSTAT | CGLO_ENERGY; - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat, cr, ir, @@ -668,11 +645,6 @@ void gmx::LegacySimulator::do_mimic() cglo_flags, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } } { diff --git a/src/gromacs/mdrun/minimize.cpp b/src/gromacs/mdrun/minimize.cpp index 497b3accd9..07e9d1c7e5 100644 --- a/src/gromacs/mdrun/minimize.cpp +++ b/src/gromacs/mdrun/minimize.cpp @@ -434,6 +434,7 @@ static void init_em(FILE* fplog, if (DOMAINDECOMP(cr)) { + // Local state only becomes valid now. dd_init_local_state(*cr->dd, state_global, &ems->s); /* Distribute the charge groups over the nodes from the master node */ @@ -1246,7 +1247,6 @@ void LegacySimulator::do_cg() { const char* CG = "Polak-Ribiere Conjugate Gradients"; - gmx_localtop_t top(top_global.ffparams); gmx_global_stat_t gstat; double tmp, minstep; real stepsize; @@ -1306,7 +1306,7 @@ void LegacySimulator::do_cg() state_global, top_global, s_min, - &top, + top, nrnb, fr, mdAtoms, @@ -1359,7 +1359,7 @@ void LegacySimulator::do_cg() cr, ms, top_global, - &top, + top, inputrec, imdSession, pull_work, @@ -1558,7 +1558,7 @@ void LegacySimulator::do_cg() imdSession, pull_work, s_min, - &top, + top, mdAtoms, fr, vsite, @@ -1673,7 +1673,7 @@ void LegacySimulator::do_cg() imdSession, pull_work, s_min, - &top, + top, mdAtoms, fr, vsite, @@ -1964,7 +1964,6 @@ void LegacySimulator::do_lbfgs() { static const char* LBFGS = "Low-Memory BFGS Minimizer"; em_state_t ems; - gmx_localtop_t top(top_global.ffparams); gmx_global_stat_t gstat; auto* mdatoms = mdAtoms->mdatoms(); @@ -2024,7 +2023,7 @@ void LegacySimulator::do_lbfgs() state_global, top_global, &ems, - &top, + top, nrnb, fr, mdAtoms, @@ -2115,7 +2114,7 @@ void LegacySimulator::do_lbfgs() cr, ms, top_global, - &top, + top, inputrec, imdSession, pull_work, @@ -2781,7 +2780,6 @@ void LegacySimulator::do_lbfgs() void LegacySimulator::do_steep() { const char* SD = "Steepest Descents"; - gmx_localtop_t top(top_global.ffparams); gmx_global_stat_t gstat; real stepsize; real ustep; @@ -2819,7 +2817,7 @@ void LegacySimulator::do_steep() state_global, top_global, s_try, - &top, + top, nrnb, fr, mdAtoms, @@ -2878,7 +2876,7 @@ void LegacySimulator::do_steep() cr, ms, top_global, - &top, + top, inputrec, imdSession, pull_work, @@ -3024,7 +3022,7 @@ void LegacySimulator::do_steep() imdSession, pull_work, s_min, - &top, + top, mdAtoms, fr, vsite, @@ -3114,7 +3112,6 @@ void LegacySimulator::do_nm() { const char* NM = "Normal Mode Analysis"; int nnodes; - gmx_localtop_t top(top_global.ffparams); gmx_global_stat_t gstat; tensor vir, pres; rvec mu_tot = { 0 }; @@ -3164,7 +3161,7 @@ void LegacySimulator::do_nm() state_global, top_global, &state_work, - &top, + top, nrnb, fr, mdAtoms, @@ -3270,7 +3267,7 @@ void LegacySimulator::do_nm() cr, ms, top_global, - &top, + top, inputrec, imdSession, pull_work, @@ -3352,7 +3349,7 @@ void LegacySimulator::do_nm() pull_work, bNS, force_flags, - &top, + top, constr, enerd, state_work.s.natoms, diff --git a/src/gromacs/mdrun/rerun.cpp b/src/gromacs/mdrun/rerun.cpp index 799b1b716a..d383df413b 100644 --- a/src/gromacs/mdrun/rerun.cpp +++ b/src/gromacs/mdrun/rerun.cpp @@ -183,7 +183,6 @@ void gmx::LegacySimulator::do_rerun() t_trxstatus* status = nullptr; rvec mu_tot; t_trxframe rerun_fr; - gmx_localtop_t top(top_global.ffparams); ForceBuffers f; gmx_global_stat_t gstat; gmx_shellfc_t* shellfc; @@ -331,14 +330,9 @@ void gmx::LegacySimulator::do_rerun() } } - // Local state only becomes valid now. - std::unique_ptr stateInstance; - t_state* state; - if (DOMAINDECOMP(cr)) { - stateInstance = std::make_unique(); - state = stateInstance.get(); + // Local state only becomes valid now. dd_init_local_state(*cr->dd, state_global, state); /* Distribute the charge groups over the nodes from the master node */ @@ -356,7 +350,7 @@ void gmx::LegacySimulator::do_rerun() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -370,7 +364,7 @@ void gmx::LegacySimulator::do_rerun() /* Copy the pointer to the global state */ state = state_global; - mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc); + mdAlgorithmsSetupAtomData(cr, *ir, top_global, top, fr, &f, mdAtoms, constr, vsite, shellfc); } auto* mdatoms = mdAtoms->mdatoms(); @@ -392,11 +386,7 @@ void gmx::LegacySimulator::do_rerun() int64_t step_rel = 0; { - int cglo_flags = CGLO_GSTAT; - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } + int cglo_flags = CGLO_GSTAT; bool bSumEkinhOld = false; t_vcm* vcm = nullptr; compute_globals(gstat, @@ -423,11 +413,6 @@ void gmx::LegacySimulator::do_rerun() cglo_flags, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } } if (MASTER(cr)) @@ -608,7 +593,7 @@ void gmx::LegacySimulator::do_rerun() state, &f, mdAtoms, - &top, + top, fr, vsite, constr, @@ -647,7 +632,7 @@ void gmx::LegacySimulator::do_rerun() pull_work, bNS, force_flags, - &top, + top, constr, enerd, state->natoms, @@ -690,7 +675,7 @@ void gmx::LegacySimulator::do_rerun() step, nrnb, wcycle, - &top, + top, state->box, state->x.arrayRefWithPadding(), &state->hist, @@ -751,10 +736,6 @@ void gmx::LegacySimulator::do_rerun() SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal); int cglo_flags = CGLO_GSTAT | CGLO_ENERGY; - if (DOMAINDECOMP(cr) && dd_localTopologyChecker(*cr->dd).shouldCheckNumberOfBondedInteractions()) - { - cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat, cr, ir, @@ -779,11 +760,6 @@ void gmx::LegacySimulator::do_rerun() cglo_flags, step, &observablesReducer); - if (DOMAINDECOMP(cr)) - { - dd_localTopologyChecker(cr->dd)->checkNumberOfBondedInteractions( - &top, makeConstArrayRef(state->x), state->box); - } } /* Note: this is OK, but there are some numerical precision issues with using the convergence of diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index ffc5db038c..17db7f419b 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -1425,15 +1425,33 @@ int Mdrunner::mdrunner() // requires it (e.g. pull, CompEl, density fitting), so that we // don't update the local atom sets unilaterally every step. LocalAtomSetManager atomSets; + + // Local state and topology are declared (and perhaps constructed) + // now, because DD needs them for the LocalTopologyChecker, but + // they do not contain valid data until after the first DD + // partition. + std::unique_ptr localStateInstance; + t_state* localState; + gmx_localtop_t localTopology(mtop.ffparams); + if (ddBuilder) { + localStateInstance = std::make_unique(); + localState = localStateInstance.get(); // TODO Pass the GPU streams to ddBuilder to use in buffer // transfers (e.g. halo exchange) - cr->dd = ddBuilder->build(&atomSets); + cr->dd = ddBuilder->build(&atomSets, localTopology, *localState, &observablesReducerBuilder); // The builder's job is done, so destruct it ddBuilder.reset(nullptr); // Note that local state still does not exist yet. } + else + { + // Without DD, the local state is merely an alias to the global state, + // so we don't need to allocate anything. + localState = globalState.get(); + } + // Ensure that all atoms within the same update group are in the // same periodic image. Otherwise, a simulation that did not use // update groups (e.g. a single-rank simulation) cannot always be @@ -2023,7 +2041,8 @@ int Mdrunner::mdrunner() GMX_ASSERT(stopHandlerBuilder_, "Runner must provide StopHandlerBuilder to simulator."); SimulatorBuilder simulatorBuilder; - simulatorBuilder.add(SimulatorStateData(globalState.get(), &observablesHistory, &enerd, &ekind)); + simulatorBuilder.add(SimulatorStateData( + globalState.get(), localState, &observablesHistory, &enerd, &ekind)); simulatorBuilder.add(std::move(membedHolder)); simulatorBuilder.add(std::move(stopHandlerBuilder_)); simulatorBuilder.add(SimulatorConfig(mdrunOptions, startingBehavior, &runScheduleWork)); @@ -2042,7 +2061,7 @@ int Mdrunner::mdrunner() simulatorBuilder.add(CenterOfMassPulling(pull_work)); // Todo move to an MDModule simulatorBuilder.add(IonSwapping(swap)); - simulatorBuilder.add(TopologyData(mtop, mdAtoms.get())); + simulatorBuilder.add(TopologyData(mtop, &localTopology, mdAtoms.get())); simulatorBuilder.add(BoxDeformationHandle(deform.get())); simulatorBuilder.add(std::move(modularSimulatorCheckpointData)); @@ -2109,6 +2128,7 @@ int Mdrunner::mdrunner() // As soon as we destroy GPU contexts after mdrunner() exits, these lines should go. mdAtoms.reset(nullptr); globalState.reset(nullptr); + localStateInstance.reset(nullptr); mdModules_.reset(nullptr); // destruct force providers here as they might also use the GPU fr.reset(nullptr); // destruct forcerec before gpu // TODO convert to C++ so we can get rid of these frees diff --git a/src/gromacs/mdrun/simulatorbuilder.cpp b/src/gromacs/mdrun/simulatorbuilder.cpp index 87cab10cfe..d53d456363 100644 --- a/src/gromacs/mdrun/simulatorbuilder.cpp +++ b/src/gromacs/mdrun/simulatorbuilder.cpp @@ -148,7 +148,9 @@ std::unique_ptr SimulatorBuilder::build(bool useModularSimulator) centerOfMassPulling_->pull_work, ionSwapping_->ionSwap, topologyData_->top_global, + topologyData_->localTopology, simulatorStateData_->globalState_p, + simulatorStateData_->localState_p, simulatorStateData_->observablesHistory_p, topologyData_->mdAtoms, profiling_->nrnb, @@ -186,7 +188,9 @@ std::unique_ptr SimulatorBuilder::build(bool useModularSimulator) centerOfMassPulling_->pull_work, ionSwapping_->ionSwap, topologyData_->top_global, + topologyData_->localTopology, simulatorStateData_->globalState_p, + simulatorStateData_->localState_p, simulatorStateData_->observablesHistory_p, topologyData_->mdAtoms, profiling_->nrnb, diff --git a/src/gromacs/mdrun/simulatorbuilder.h b/src/gromacs/mdrun/simulatorbuilder.h index 5ab848fa80..787dc721d1 100644 --- a/src/gromacs/mdrun/simulatorbuilder.h +++ b/src/gromacs/mdrun/simulatorbuilder.h @@ -48,6 +48,7 @@ class energyhistory_t; class gmx_ekindata_t; struct gmx_enerdata_t; struct gmx_enfrot; +struct gmx_localtop_t; struct gmx_mtop_t; struct gmx_multisim_t; struct gmx_output_env_t; @@ -117,10 +118,12 @@ struct SimulatorStateData { //! Build collection of current state data. SimulatorStateData(t_state* globalState, + t_state* localState, ObservablesHistory* observablesHistory, gmx_enerdata_t* enerdata, gmx_ekindata_t* ekindata) : globalState_p(globalState), + localState_p(localState), observablesHistory_p(observablesHistory), enerdata_p(enerdata), ekindata_p(ekindata) @@ -132,6 +135,8 @@ struct SimulatorStateData //! Handle to global state of the simulation. t_state* globalState_p; + //! Handle to local state of the simulation. + t_state* localState_p; //! Handle to current simulation history. ObservablesHistory* observablesHistory_p; //! Handle to collected data for energy groups. @@ -295,13 +300,15 @@ class TopologyData { public: //! Build collection from simulation data. - TopologyData(const gmx_mtop_t& globalTopology, MDAtoms* mdAtoms) : - top_global(globalTopology), mdAtoms(mdAtoms) + TopologyData(const gmx_mtop_t& globalTopology, gmx_localtop_t* localTopology, MDAtoms* mdAtoms) : + top_global(globalTopology), localTopology(localTopology), mdAtoms(mdAtoms) { } //! Handle to global simulation topology. const gmx_mtop_t& top_global; + //! Handle to local simulation topology. + gmx_localtop_t* localTopology; //! Handle to information about MDAtoms. MDAtoms* mdAtoms; }; diff --git a/src/gromacs/mdrun/tpi.cpp b/src/gromacs/mdrun/tpi.cpp index 0827b4a03d..68700531b3 100644 --- a/src/gromacs/mdrun/tpi.cpp +++ b/src/gromacs/mdrun/tpi.cpp @@ -166,7 +166,6 @@ void LegacySimulator::do_tpi() GMX_RELEASE_ASSERT(gmx_omp_nthreads_get(ModuleMultiThread::Default) == 1, "TPI does not support OpenMP"); - gmx_localtop_t top(top_global.ffparams); gmx::ForceBuffers f; real lambda, t, temp, beta, drmax, epot; double embU, sum_embU, *sum_UgembU, V, V_all, VembU_all; @@ -222,7 +221,7 @@ void LegacySimulator::do_tpi() nnodes = cr->nnodes; - gmx_mtop_generate_local_top(top_global, &top, inputrec->efep != FreeEnergyPerturbationType::No); + gmx_mtop_generate_local_top(top_global, top, inputrec->efep != FreeEnergyPerturbationType::No); const SimulationGroups* groups = &top_global.groups; @@ -698,7 +697,7 @@ void LegacySimulator::do_tpi() gmx::constArrayRefFromArray(mdatoms->chargeA, mdatoms->nr), fr->atomInfo); - fr->nbv->constructPairlist(InteractionLocality::Local, top.excls, step, nrnb); + fr->nbv->constructPairlist(InteractionLocality::Local, top->excls, step, nrnb); bNS = FALSE; } @@ -780,7 +779,7 @@ void LegacySimulator::do_tpi() step, nrnb, wcycle, - &top, + top, state_global->box, state_global->x.arrayRefWithPadding(), &state_global->hist, diff --git a/src/gromacs/modularsimulator/computeglobalselement.cpp b/src/gromacs/modularsimulator/computeglobalselement.cpp index f54747211f..97634447ba 100644 --- a/src/gromacs/modularsimulator/computeglobalselement.cpp +++ b/src/gromacs/modularsimulator/computeglobalselement.cpp @@ -44,7 +44,6 @@ #include "computeglobalselement.h" #include "gromacs/domdec/domdec.h" -#include "gromacs/domdec/localtopologychecker.h" #include "gromacs/gmxlib/network.h" #include "gromacs/gmxlib/nrnb.h" #include "gromacs/math/vec.h" @@ -94,7 +93,6 @@ ComputeGlobalsElement::ComputeGlobalsElement(StatePropagatorData* sta nullSignaller_(std::make_unique(nullptr, nullptr, nullptr, false, false)), statePropagatorData_(statePropagatorData), energyData_(energyData), - localTopology_(nullptr), freeEnergyPerturbationData_(freeEnergyPerturbationData), vcm_(global_top.groups, *inputrec), signals_(signals), @@ -123,8 +121,6 @@ ComputeGlobalsElement::~ComputeGlobalsElement() template void ComputeGlobalsElement::elementSetup() { - GMX_ASSERT(localTopology_, "Setup called before local topology was set."); - if (doStopCM_ && !inputrec_->bContinuation) { // To minimize communication, compute_globals computes the COM velocity @@ -286,10 +282,6 @@ void ComputeGlobalsElement::compute(gmx::Step step, const auto* lastbox = useLastBox ? statePropagatorData_->constPreviousBox() : statePropagatorData_->constBox(); - if (DOMAINDECOMP(cr_) && dd_localTopologyChecker(*cr_->dd).shouldCheckNumberOfBondedInteractions()) - { - flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS; - } compute_globals(gstat_, cr_, inputrec_, @@ -315,10 +307,6 @@ void ComputeGlobalsElement::compute(gmx::Step step, flags, step, observablesReducer_); - if (DOMAINDECOMP(cr_)) - { - dd_localTopologyChecker(cr_->dd)->checkNumberOfBondedInteractions(localTopology_, x, box); - } if (flags & CGLO_STOPCM && !isInit) { process_and_stopcm_grp(fplog_, &vcm_, *mdAtoms_->mdatoms(), x, v); @@ -326,12 +314,6 @@ void ComputeGlobalsElement::compute(gmx::Step step, } } -template -void ComputeGlobalsElement::setTopology(const gmx_localtop_t* top) -{ - localTopology_ = top; -} - template std::optional ComputeGlobalsElement::registerEnergyCallback(EnergySignallerEvent event) { diff --git a/src/gromacs/modularsimulator/computeglobalselement.h b/src/gromacs/modularsimulator/computeglobalselement.h index f6905f20c7..dbd4952c90 100644 --- a/src/gromacs/modularsimulator/computeglobalselement.h +++ b/src/gromacs/modularsimulator/computeglobalselement.h @@ -98,11 +98,7 @@ typedef std::function CheckBondedInteractionsCallback; * \tparam algorithm The global reduction scheme */ template -class ComputeGlobalsElement final : - public ISimulatorElement, - public IEnergySignallerClient, - public ITrajectorySignallerClient, - public ITopologyHolderClient +class ComputeGlobalsElement final : public ISimulatorElement, public IEnergySignallerClient, public ITrajectorySignallerClient { public: //! Constructor @@ -171,8 +167,6 @@ public: ObservablesReducer* observablesReducer); private: - //! ITopologyClient implementation - void setTopology(const gmx_localtop_t* top) override; //! IEnergySignallerClient implementation std::optional registerEnergyCallback(EnergySignallerEvent event) override; //! ITrajectorySignallerClient implementation @@ -236,8 +230,6 @@ private: StatePropagatorData* statePropagatorData_; //! Pointer to the energy data (needed for the tensors and mu_tot) EnergyData* energyData_; - //! Pointer to the local topology (only needed for checkNumberOfBondedInteractions) - const gmx_localtop_t* localTopology_; //! Pointer to the free energy perturbation data FreeEnergyPerturbationData* freeEnergyPerturbationData_; diff --git a/src/gromacs/modularsimulator/domdechelper.cpp b/src/gromacs/modularsimulator/domdechelper.cpp index d268d1dff7..30d3d4ea15 100644 --- a/src/gromacs/modularsimulator/domdechelper.cpp +++ b/src/gromacs/modularsimulator/domdechelper.cpp @@ -120,8 +120,8 @@ void DomDecHelper::run(Step step, Time gmx_unused time) { return; } - std::unique_ptr localState = statePropagatorData_->localState(); - t_state* globalState = statePropagatorData_->globalState(); + t_state* localState = statePropagatorData_->localState(); + t_state* globalState = statePropagatorData_->globalState(); // constant choices for this call to dd_partition_system const bool verbose = isVerbose_ && (step % verbosePrintInterval_ == 0 || step == inputrec_->init_step); @@ -139,19 +139,19 @@ void DomDecHelper::run(Step step, Time gmx_unused time) } if (isMasterState) { - dd_collect_state(cr_->dd, localState.get(), globalState); + dd_collect_state(cr_->dd, localState, globalState); } // Distribute the charge groups over the nodes from the master node - partitionSystem(verbose, isMasterState, nstglobalcomm_, wcycle_, std::move(localState), globalState); + partitionSystem(verbose, isMasterState, nstglobalcomm_, wcycle_, localState, globalState); } -void DomDecHelper::partitionSystem(bool verbose, - bool isMasterState, - int nstglobalcomm, - gmx_wallcycle* wcycle, - std::unique_ptr localState, - t_state* globalState) +void DomDecHelper::partitionSystem(bool verbose, + bool isMasterState, + int nstglobalcomm, + gmx_wallcycle* wcycle, + t_state* localState, + t_state* globalState) { ForceBuffers* forcePointer = statePropagatorData_->forcePointer(); @@ -170,17 +170,17 @@ void DomDecHelper::partitionSystem(bool verbose, *inputrec_, imdSession_, pull_work_, - localState.get(), + localState, forcePointer, mdAtoms_, - topologyHolder_->localTopology_.get(), + topologyHolder_->localTopology_, fr_, vsite_, constr_, nrnb_, wcycle, verbose); - statePropagatorData_->setLocalState(std::move(localState)); + statePropagatorData_->setLocalState(localState); for (const auto& callback : domdecCallbacks_) { callback(); diff --git a/src/gromacs/modularsimulator/domdechelper.h b/src/gromacs/modularsimulator/domdechelper.h index 15f0fd76ab..1023d39251 100644 --- a/src/gromacs/modularsimulator/domdechelper.h +++ b/src/gromacs/modularsimulator/domdechelper.h @@ -141,12 +141,12 @@ private: TopologyHolder* topologyHolder_; //! Helper function unifying the DD partitioning calls in setup() and run() - void partitionSystem(bool verbose, - bool isMasterState, - int nstglobalcomm, - gmx_wallcycle* wcycle, - std::unique_ptr localState, - t_state* globalState); + void partitionSystem(bool verbose, + bool isMasterState, + int nstglobalcomm, + gmx_wallcycle* wcycle, + t_state* localState, + t_state* globalState); // Access to ISimulator data //! Handles logging. diff --git a/src/gromacs/modularsimulator/simulatoralgorithm.cpp b/src/gromacs/modularsimulator/simulatoralgorithm.cpp index 8ef941fd18..2d127f41ef 100644 --- a/src/gromacs/modularsimulator/simulatoralgorithm.cpp +++ b/src/gromacs/modularsimulator/simulatoralgorithm.cpp @@ -428,6 +428,7 @@ ModularSimulatorAlgorithmBuilder::ModularSimulatorAlgorithmBuilder( legacySimulatorData->fplog, legacySimulatorData->cr, legacySimulatorData->state_global, + legacySimulatorData->state, legacySimulatorData->fr->nbv->useGpu(), legacySimulatorData->fr->bMolPBC, legacySimulatorData->mdrunOptions.writeConfout, @@ -544,6 +545,7 @@ ModularSimulatorAlgorithm ModularSimulatorAlgorithmBuilder::build() // Build topology holder algorithm.topologyHolder_ = topologyHolderBuilder_.build(legacySimulatorData_->top_global, + legacySimulatorData_->top, legacySimulatorData_->cr, legacySimulatorData_->inputrec, legacySimulatorData_->fr, diff --git a/src/gromacs/modularsimulator/statepropagatordata.cpp b/src/gromacs/modularsimulator/statepropagatordata.cpp index a0177f43c9..c57ec7c061 100644 --- a/src/gromacs/modularsimulator/statepropagatordata.cpp +++ b/src/gromacs/modularsimulator/statepropagatordata.cpp @@ -146,6 +146,7 @@ StatePropagatorData::StatePropagatorData(int numAtoms, FILE* fplog, const t_commrec* cr, t_state* globalState, + t_state* localState, bool useGPU, bool canMoleculesBeDistributedOverPBC, bool writeFinalConfiguration, @@ -180,10 +181,9 @@ StatePropagatorData::StatePropagatorData(int numAtoms, // Local state only becomes valid now. if (DOMAINDECOMP(cr)) { - auto localState = std::make_unique(); - dd_init_local_state(*cr->dd, globalState, localState.get()); + dd_init_local_state(*cr->dd, globalState, localState); stateHasVelocities = ((localState->flags & enumValueToBitMask(StateEntry::V)) != 0); - setLocalState(std::move(localState)); + setLocalState(localState); } else { @@ -327,29 +327,41 @@ int StatePropagatorData::totalNumAtoms() const return totalNumAtoms_; } -std::unique_ptr StatePropagatorData::localState() +t_state* StatePropagatorData::localState() { - auto state = std::make_unique(); - state->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V) - | enumValueToBitMask(StateEntry::Box); - state_change_natoms(state.get(), localNAtoms_); - state->x = x_; - state->v = v_; - copy_mat(box_, state->box); - state->ddp_count = ddpCount_; - state->ddp_count_cg_gl = ddpCountCgGl_; - state->cg_gl = cgGl_; - return state; + localState_->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V) + | enumValueToBitMask(StateEntry::Box); + state_change_natoms(localState_, localNAtoms_); + std::swap(localState_->x, x_); + std::swap(localState_->v, v_); + copy_mat(box_, localState_->box); + localState_->ddp_count = ddpCount_; + localState_->ddp_count_cg_gl = ddpCountCgGl_; + localState_->cg_gl = cgGl_; + return localState_; } -void StatePropagatorData::setLocalState(std::unique_ptr state) +std::unique_ptr StatePropagatorData::copyLocalState(std::unique_ptr copy) { + copy->flags = enumValueToBitMask(StateEntry::X) | enumValueToBitMask(StateEntry::V) + | enumValueToBitMask(StateEntry::Box); + state_change_natoms(copy.get(), localNAtoms_); + copy->x = x_; + copy->v = v_; + copy_mat(box_, copy->box); + copy->ddp_count = ddpCount_; + copy->ddp_count_cg_gl = ddpCountCgGl_; + copy->cg_gl = cgGl_; + return copy; +} + +void StatePropagatorData::setLocalState(t_state* state) +{ + localState_ = state; localNAtoms_ = state->natoms; - x_.resizeWithPadding(localNAtoms_); previousX_.resizeWithPadding(localNAtoms_); - v_.resizeWithPadding(localNAtoms_); - x_ = state->x; - v_ = state->v; + std::swap(x_, state->x); + std::swap(v_, state->v); copy_mat(state->box, box_); copyPosition(); ddpCount_ = state->ddp_count; @@ -433,8 +445,9 @@ void StatePropagatorData::Element::scheduleTask(Step step, void StatePropagatorData::Element::saveState() { - GMX_ASSERT(!localStateBackup_, "Save state called again before previous state was written."); - localStateBackup_ = statePropagatorData_->localState(); + GMX_ASSERT(!localStateBackupValid_, + "Save state called again before previous state was written."); + localStateBackup_ = statePropagatorData_->copyLocalState(std::move(localStateBackup_)); if (freeEnergyPerturbationData_) { localStateBackup_->fep_state = freeEnergyPerturbationData_->currentFEPState(); @@ -443,6 +456,7 @@ void StatePropagatorData::Element::saveState() localStateBackup_->flags |= enumValueToBitMask(StateEntry::Lambda) | enumValueToBitMask(StateEntry::FepState); } + localStateBackupValid_ = true; } std::optional StatePropagatorData::Element::registerTrajectorySignallerCallback(TrajectoryEvent event) @@ -511,7 +525,7 @@ void StatePropagatorData::Element::write(gmx_mdoutf_t outf, Step currentStep, Ti wallcycle_stop(mdoutf_get_wcycle(outf), WallCycleCounter::Traj); return; } - GMX_ASSERT(localStateBackup_, "Trajectory writing called, but no state saved."); + GMX_ASSERT(localStateBackupValid_, "Trajectory writing called, but no state saved."); // TODO: This is only used for CPT - needs to be filled when we turn CPT back on ObservablesHistory* observablesHistory = nullptr; @@ -531,7 +545,7 @@ void StatePropagatorData::Element::write(gmx_mdoutf_t outf, Step currentStep, Ti if (currentStep != lastStep_ || !isRegularSimulationEnd_) { - localStateBackup_.reset(); + localStateBackupValid_ = false; } wallcycle_stop(mdoutf_get_wcycle(outf), WallCycleCounter::Traj); } @@ -695,7 +709,7 @@ void StatePropagatorData::Element::trajectoryWriterTeardown(gmx_mdoutf* gmx_unus return; } - GMX_ASSERT(localStateBackup_, "Final trajectory writing called, but no state saved."); + GMX_ASSERT(localStateBackupValid_, "Final trajectory writing called, but no state saved."); wallcycle_start(mdoutf_get_wcycle(outf), WallCycleCounter::Traj); if (DOMAINDECOMP(cr_)) @@ -770,6 +784,7 @@ StatePropagatorData::Element::Element(StatePropagatorData* statePropagatorData, nstvout_(nstvout), nstfout_(nstfout), nstxout_compressed_(nstxout_compressed), + localStateBackup_(std::make_unique()), writeOutStep_(-1), freeEnergyPerturbationData_(nullptr), isRegularSimulationEnd_(false), diff --git a/src/gromacs/modularsimulator/statepropagatordata.h b/src/gromacs/modularsimulator/statepropagatordata.h index d0f4902765..734a0708a9 100644 --- a/src/gromacs/modularsimulator/statepropagatordata.h +++ b/src/gromacs/modularsimulator/statepropagatordata.h @@ -105,6 +105,7 @@ public: FILE* fplog, const t_commrec* cr, t_state* globalState, + t_state* localState, bool useGPU, bool canMoleculesBeDistributedOverPBC, bool writeFinalConfiguration, @@ -220,10 +221,18 @@ private: void doCheckpointData(CheckpointData* checkpointData); // Access to legacy state - //! Get a deep copy of the current state in legacy format - std::unique_ptr localState(); - //! Update the current state with a state in legacy format - void setLocalState(std::unique_ptr state); + //! Give ownership of local state resources in legacy format + t_state* localState(); + //! Take ownership of local state resources in legacy format + void setLocalState(t_state* state); + /*! \brief Deep copy the local state into the provided copy and + * return it + * + * In order to minimize reallocations, this function takes as a sink + * a local state object owned by the caller, copies the current local + * state into it, and returns the same object via a move. + */ + std::unique_ptr copyLocalState(std::unique_ptr copy); //! Get a pointer to the global state t_state* globalState(); //! Get a force pointer @@ -244,6 +253,8 @@ private: // Access to ISimulator data //! Full simulation state (only non-nullptr on master rank). t_state* globalState_; + //! Local simulation state + t_state* localState_; }; /*! \internal @@ -362,6 +373,11 @@ private: //! Pointer to keep a backup of the state for later writeout std::unique_ptr localStateBackup_; + /*! \brief Whether the contents of localStateBackup_ are logically valid + * + * This ensures that we don't make a second backup without consuming the + * first. */ + bool localStateBackupValid_ = false; //! Step at which next writeout occurs Step writeOutStep_; //! Backup current state diff --git a/src/gromacs/modularsimulator/topologyholder.cpp b/src/gromacs/modularsimulator/topologyholder.cpp index ec3781db44..d697a2c96b 100644 --- a/src/gromacs/modularsimulator/topologyholder.cpp +++ b/src/gromacs/modularsimulator/topologyholder.cpp @@ -52,15 +52,14 @@ namespace gmx { TopologyHolder::TopologyHolder(std::vector clients, const gmx_mtop_t& globalTopology, + gmx_localtop_t* localTopology, const t_commrec* cr, const t_inputrec* inputrec, t_forcerec* fr, MDAtoms* mdAtoms, Constraints* constr, VirtualSitesHandler* vsite) : - globalTopology_(globalTopology), - localTopology_(std::make_unique(globalTopology.ffparams)), - clients_(std::move(clients)) + globalTopology_(globalTopology), localTopology_(localTopology), clients_(std::move(clients)) { if (!DOMAINDECOMP(cr)) { @@ -71,7 +70,7 @@ TopologyHolder::TopologyHolder(std::vector clients, // TopologyHolder has no access to the forces, so we are passing a nullptr // TODO: Find a unique approach to resizing the forces in modular simulator (#3461) mdAlgorithmsSetupAtomData( - cr, *inputrec, globalTopology, localTopology_.get(), fr, nullptr, mdAtoms, constr, vsite, nullptr); + cr, *inputrec, globalTopology, localTopology_, fr, nullptr, mdAtoms, constr, vsite, nullptr); } // Send copy of initial topology to clients updateLocalTopology(); @@ -86,7 +85,7 @@ void TopologyHolder::updateLocalTopology() { for (auto& client : clients_) { - client->setTopology(localTopology_.get()); + client->setTopology(localTopology_); } } DomDecCallback TopologyHolder::registerDomDecCallback() diff --git a/src/gromacs/modularsimulator/topologyholder.h b/src/gromacs/modularsimulator/topologyholder.h index 8b0316043e..3c334c272c 100644 --- a/src/gromacs/modularsimulator/topologyholder.h +++ b/src/gromacs/modularsimulator/topologyholder.h @@ -76,6 +76,7 @@ public: //! Constructor TopologyHolder(std::vector clients, const gmx_mtop_t& globalTopology, + gmx_localtop_t* localTopology, const t_commrec* cr, const t_inputrec* inputrec, t_forcerec* fr, @@ -99,7 +100,7 @@ private: //! Constant reference to the global topology const gmx_mtop_t& globalTopology_; //! Pointer to the currently valid local topology - std::unique_ptr localTopology_; + gmx_localtop_t* localTopology_; //! List of clients to be updated if local topology changes std::vector clients_;