2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2019,2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * \brief Defines the domain decomposition helper for the modular simulator
38 * \author Pascal Merz <pascal.merz@me.com>
39 * \ingroup module_modularsimulator
44 #include "domdechelper.h"
46 #include "gromacs/domdec/collect.h"
47 #include "gromacs/domdec/partition.h"
48 #include "gromacs/mdtypes/commrec.h"
49 #include "gromacs/mdtypes/inputrec.h"
50 #include "gromacs/mdtypes/state.h"
51 #include "gromacs/pbcutil/pbc.h"
53 #include "freeenergyperturbationdata.h"
54 #include "statepropagatordata.h"
55 #include "topologyholder.h"
59 DomDecHelper::DomDecHelper(bool isVerbose,
60 int verbosePrintInterval,
61 StatePropagatorData* statePropagatorData,
62 FreeEnergyPerturbationData* freeEnergyPerturbationData,
63 TopologyHolder* topologyHolder,
64 CheckBondedInteractionsCallback checkBondedInteractionsCallback,
68 const MDLogger& mdlog,
73 gmx_wallcycle* wcycle,
75 VirtualSitesHandler* vsite,
76 ImdSession* imdSession,
79 isVerbose_(isVerbose),
80 verbosePrintInterval_(verbosePrintInterval),
81 nstglobalcomm_(nstglobalcomm),
82 statePropagatorData_(statePropagatorData),
83 freeEnergyPerturbationData_(freeEnergyPerturbationData),
84 topologyHolder_(topologyHolder),
85 checkBondedInteractionsCallback_(std::move(checkBondedInteractionsCallback)),
96 imdSession_(imdSession),
99 GMX_ASSERT(DOMAINDECOMP(cr), "Domain decomposition Helper constructed in non-DD simulation");
102 void DomDecHelper::setup()
104 // constant choices for this call to dd_partition_system
105 const bool verbose = false;
106 const bool isMasterState = true;
107 const int nstglobalcomm = 1;
108 gmx_wallcycle* wcycle = nullptr;
110 // Distribute the charge groups over the nodes from the master node
111 partitionSystem(verbose, isMasterState, nstglobalcomm, wcycle,
112 statePropagatorData_->localState(), statePropagatorData_->globalState());
115 void DomDecHelper::run(Step step, Time gmx_unused time)
117 if (step != nextNSStep_ || (step == inputrec_->init_step && inputrec_->bContinuation))
121 std::unique_ptr<t_state> localState = statePropagatorData_->localState();
122 t_state* globalState = statePropagatorData_->globalState();
124 // constant choices for this call to dd_partition_system
125 const bool verbose = isVerbose_ && (step % verbosePrintInterval_ == 0 || step == inputrec_->init_step);
126 bool isMasterState = false;
128 // Correct the new box if it is too skewed
129 if (inputrecDynamicBox(inputrec_))
131 // TODO: Correcting the box is done here (if using DD) or in ForceElement (non-DD simulations).
132 // Think about unifying this responsibility, could this be done in one place?
133 if (correct_box(fplog_, step, localState->box))
135 isMasterState = true;
140 dd_collect_state(cr_->dd, localState.get(), globalState);
143 // Distribute the charge groups over the nodes from the master node
144 partitionSystem(verbose, isMasterState, nstglobalcomm_, wcycle_, std::move(localState), globalState);
147 void DomDecHelper::partitionSystem(bool verbose,
150 gmx_wallcycle* wcycle,
151 std::unique_ptr<t_state> localState,
152 t_state* globalState)
154 ForceBuffers* forcePointer = statePropagatorData_->forcePointer();
156 // Distribute the charge groups over the nodes from the master node
157 dd_partition_system(fplog_, mdlog_, inputrec_->init_step, cr_, isMasterState, nstglobalcomm,
158 globalState, topologyHolder_->globalTopology(), inputrec_, imdSession_,
159 pull_work_, localState.get(), forcePointer, mdAtoms_,
160 topologyHolder_->localTopology_.get(), fr_, vsite_, constr_, nrnb_, wcycle,
162 topologyHolder_->updateLocalTopology();
163 checkBondedInteractionsCallback_();
164 statePropagatorData_->setLocalState(std::move(localState));
165 if (freeEnergyPerturbationData_)
167 freeEnergyPerturbationData_->updateMDAtoms();
171 std::optional<SignallerCallback> DomDecHelper::registerNSCallback()
173 return [this](Step step, Time gmx_unused time) { this->nextNSStep_ = step; };