std::vector<real> chargeA{ 1 };
md.homenr = ssize(chargeA);
md.chargeA = chargeA.data();
- CommrecHandle cr = init_commrec(MPI_COMM_WORLD, nullptr);
+ CommrecHandle cr = init_commrec(MPI_COMM_WORLD);
matrix boxDummy = { { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 } };
ForceProviderInput forceProviderInput({}, md, 0.0, boxDummy, *cr);
int numSharingSimulations = 1;
if (awhParams.shareBiasMultisim && isMultiSim(multiSimRecord_))
{
- numSharingSimulations = multiSimRecord_->nsim;
+ numSharingSimulations = multiSimRecord_->numSimulations_;
}
/* Initialize all the biases */
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
const std::vector<size_t>& pointSize,
const gmx_multisim_t* multiSimComm)
{
- const int numSim = multiSimComm->nsim;
+ const int numSim = multiSimComm->numSimulations_;
/* We currently enforce subsequent shared biases to have consecutive
* share-group values starting at 1. This means we can reduce shared
}
}
std::vector<int> numShareAll(numSim);
- numShareAll[multiSimComm->sim] = numShare;
+ numShareAll[multiSimComm->simulationIndex_] = numShare;
gmx_sumi_sim(numShareAll.size(), numShareAll.data(), multiSimComm);
for (int sim = 1; sim < numSim; sim++)
{
}
std::vector<int> intervals(numSim * 2);
- intervals[numSim * 0 + multiSimComm->sim] = awhParams.nstSampleCoord;
- intervals[numSim * 1 + multiSimComm->sim] = awhParams.numSamplesUpdateFreeEnergy;
+ intervals[numSim * 0 + multiSimComm->simulationIndex_] = awhParams.nstSampleCoord;
+ intervals[numSim * 1 + multiSimComm->simulationIndex_] = awhParams.numSamplesUpdateFreeEnergy;
gmx_sumi_sim(intervals.size(), intervals.data(), multiSimComm);
for (int sim = 1; sim < numSim; sim++)
{
if (awhParams.awhBiasParams[b].shareGroup > 0)
{
std::vector<int64_t> pointSizes(numSim);
- pointSizes[multiSimComm->sim] = pointSize[b];
+ pointSizes[multiSimComm->simulationIndex_] = pointSize[b];
gmx_sumli_sim(pointSizes.size(), pointSizes.data(), multiSimComm);
for (int sim = 1; sim < numSim; sim++)
{
{
return;
}
- GMX_ASSERT(multiSimComm != nullptr && numSharedUpdate % multiSimComm->nsim == 0,
- "numSharedUpdate should be a multiple of multiSimComm->nsim");
- GMX_ASSERT(numSharedUpdate == multiSimComm->nsim,
+ GMX_ASSERT(multiSimComm != nullptr && numSharedUpdate % multiSimComm->numSimulations_ == 0,
+ "numSharedUpdate should be a multiple of multiSimComm->numSimulations_");
+ GMX_ASSERT(numSharedUpdate == multiSimComm->numSimulations_,
"Sharing within a simulation is not implemented (yet)");
std::vector<double> buffer(pointState.size());
/* Sum histograms over multiple simulations if needed. */
if (numSharedUpdate > 1)
{
- GMX_ASSERT(numSharedUpdate == multiSimComm->nsim,
+ GMX_ASSERT(numSharedUpdate == multiSimComm->numSimulations_,
"Sharing within a simulation is not implemented (yet)");
/* Collect the weights and counts in linear arrays to be able to use gmx_sumd_sim. */
#include <cstring>
#include "gromacs/commandline/filenm.h"
-#include "gromacs/mdrunutility/multisim.h"
#include "gromacs/mdtypes/commrec.h"
#include "gromacs/utility/basenetwork.h"
#include "gromacs/utility/cstringutil.h"
#include "gromacs/utility/futil.h"
#include "gromacs/utility/gmxassert.h"
#include "gromacs/utility/gmxmpi.h"
-#include "gromacs/utility/mpiinplacebuffers.h"
#include "gromacs/utility/real.h"
#include "gromacs/utility/smalloc.h"
/* The source code in this file should be thread-safe.
Please keep it that way. */
-CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms)
+CommrecHandle init_commrec(MPI_Comm communicator)
{
CommrecHandle handle;
t_commrec* cr;
sizeOfCommunicator = 1;
#endif
- if (ms != nullptr)
- {
-#if GMX_MPI
- MPI_Comm_split(communicator, ms->sim, rankInCommunicator, &cr->mpiDefaultCommunicator);
- cr->sizeOfDefaultCommunicator = sizeOfCommunicator / ms->nsim;
- MPI_Comm_rank(cr->mpiDefaultCommunicator, &cr->rankInDefaultCommunicator);
-#else
- gmx_fatal(FARGS, "Multisim can only run with MPI.");
-#endif
- }
- else
- {
- cr->mpiDefaultCommunicator = communicator;
- cr->sizeOfDefaultCommunicator = sizeOfCommunicator;
- cr->rankInDefaultCommunicator = rankInCommunicator;
- }
+ cr->mpiDefaultCommunicator = communicator;
+ cr->sizeOfDefaultCommunicator = sizeOfCommunicator;
+ cr->rankInDefaultCommunicator = rankInCommunicator;
+
// For now, we want things to go horribly wrong if this is used too early...
// TODO: Remove when communicators are removed from commrec (#2395)
cr->nnodes = -1;
// TODO cr->duty should not be initialized here
cr->duty = (DUTY_PP | DUTY_PME);
-#if GMX_MPI && !MPI_IN_PLACE_EXISTS
- /* initialize the MPI_IN_PLACE replacement buffers */
- snew(cr->mpb, 1);
- cr->mpb->ibuf = nullptr;
- cr->mpb->libuf = nullptr;
- cr->mpb->fbuf = nullptr;
- cr->mpb->dbuf = nullptr;
- cr->mpb->ibuf_alloc = 0;
- cr->mpb->libuf_alloc = 0;
- cr->mpb->fbuf_alloc = 0;
- cr->mpb->dbuf_alloc = 0;
-#endif
-
return handle;
}
// TODO: implement
// done_domdec(cr->dd);
}
- done_mpi_in_place_buf(cr->mpb);
}
#if GMX_MPI
// TODO We need to be able to free communicators, but the
#include "gromacs/utility/stringutil.h"
#include "gromacs/utility/unique_cptr.h"
-struct gmx_multisim_t;
struct t_commrec;
struct t_filenm;
using CommrecHandle = gmx::unique_cptr<t_commrec, done_commrec>;
//! Allocate, initialize and return the commrec.
-CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms);
+CommrecHandle init_commrec(MPI_Comm communicator);
struct t_commrec* reinitialize_commrec_for_this_thread(const t_commrec* cro);
gmx_bcast(sizeof(int), &dd->nsystems, communicator);
/* We use to allow any value of nsystems which was a divisor
- * of ms->nsim. But this required an extra communicator which
+ * of ms->numSimulations_. But this required an extra communicator which
* pulled in mpi.h in nearly all C files.
*/
- if (!(ms->nsim == 1 || ms->nsim == dd->nsystems))
+ if (!(ms->numSimulations_ == 1 || ms->numSimulations_ == dd->nsystems))
{
gmx_fatal(FARGS,
"GMX_DISRE_ENSEMBLE_SIZE (%d) is not equal to 1 or the number of systems "
"(option -multidir) %d",
- dd->nsystems, ms->nsim);
+ dd->nsystems, ms->numSimulations_);
}
if (fplog)
{
fprintf(fplog, "Our ensemble consists of systems:");
for (int i = 0; i < dd->nsystems; i++)
{
- fprintf(fplog, " %d", (ms->sim / dd->nsystems) * dd->nsystems + i);
+ fprintf(fplog, " %d", (ms->simulationIndex_ / dd->nsystems) * dd->nsystems + i);
}
fprintf(fplog, "\n");
}
if (ms)
{
- fprintf(fplog, " the orientation restraints are ensemble averaged over %d systems\n", ms->nsim);
+ fprintf(fplog, " the orientation restraints are ensemble averaged over %d systems\n",
+ ms->numSimulations_);
check_multi_int(fplog, ms, od->nr, "the number of orientation restraints", FALSE);
check_multi_int(fplog, ms, od->nref, "the number of fit atoms for orientation restraining", FALSE);
if (ms)
{
- invn = 1.0 / ms->nsim;
+ invn = 1.0 / ms->numSimulations_;
}
else
{
std::string simMesg;
if (isMultiSim(ms))
{
- simMesg += gmx::formatString(" in simulation %d", ms->sim);
+ simMesg += gmx::formatString(" in simulation %d", ms->simulationIndex_);
}
fprintf(stderr,
"\nStep %" PRId64
GMX_RELEASE_ASSERT(ms, "Invalid use of multi-simulation pointer");
- snew(buf, ms->nsim);
+ snew(buf, ms->numSimulations_);
/* send our value to all other master ranks, receive all of theirs */
- buf[ms->sim] = value;
- gmx_sumli_sim(ms->nsim, buf, ms);
+ buf[ms->simulationIndex_] = value;
+ gmx_sumli_sim(ms->numSimulations_, buf, ms);
- for (int s = 0; s < ms->nsim; s++)
+ for (int s = 0; s < ms->numSimulations_; s++)
{
if (buf[s] != value)
{
gmx_bool bPos, bEqual;
int s, d;
- snew(buf, ms->nsim);
- buf[ms->sim] = n;
- gmx_sumi_sim(ms->nsim, buf, ms);
+ snew(buf, ms->numSimulations_);
+ buf[ms->simulationIndex_] = n;
+ gmx_sumi_sim(ms->numSimulations_, buf, ms);
bPos = TRUE;
bEqual = TRUE;
- for (s = 0; s < ms->nsim; s++)
+ for (s = 0; s < ms->numSimulations_; s++)
{
bPos = bPos && (buf[s] > 0);
bEqual = bEqual && (buf[s] == buf[0]);
for (d = 2; d < nmin; d++)
{
s = 0;
- while (s < ms->nsim && d % buf[s] == 0)
+ while (s < ms->numSimulations_ && d % buf[s] == 0)
{
s++;
}
- if (s == ms->nsim)
+ if (s == ms->numSimulations_)
{
/* We found the LCM and it is less than nmin */
nmin = d;
gmx::IMDOutputProvider* outputProvider;
const gmx::MdModulesNotifier* mdModulesNotifier;
bool simulationsShareState;
- MPI_Comm mpiCommMasters;
+ MPI_Comm mastersComm;
};
of->simulationsShareState = simulationsShareState;
if (of->simulationsShareState)
{
- of->mpiCommMasters = ms->mpi_comm_masters;
+ of->mastersComm = ms->mastersComm_;
}
if (MASTER(cr))
DOMAINDECOMP(cr) ? cr->dd->nnodes : cr->nnodes, of->eIntegrator,
of->simulation_part, of->bExpanded, of->elamstats, step, t,
state_global, observablesHistory, *(of->mdModulesNotifier),
- of->simulationsShareState, of->mpiCommMasters);
+ of->simulationsShareState, of->mastersComm);
}
if (mdof_flags & (MDOF_X | MDOF_V | MDOF_F))
cr.dd = nullptr;
// Multi-sim record
- gmx_multisim_t ms;
+ gmx_multisim_t ms{ 1, 0, MPI_COMM_NULL, MPI_COMM_NULL };
// Make blocka structure for faster LINCS setup
std::vector<ListOfLists<int>> at2con_mt;
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2011-2019, by the GROMACS development team, led by
+ * Copyright (c) 2011-2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
gmx_bool bDiff;
int s;
- snew(qall, ms->nsim);
+ snew(qall, ms->numSimulations_);
qall[re->repl] = q;
- gmx_sum_sim(ms->nsim, qall, ms);
+ gmx_sum_sim(ms->numSimulations_, qall, ms);
bDiff = FALSE;
- for (s = 1; s < ms->nsim; s++)
+ for (s = 1; s < ms->numSimulations_; s++)
{
if (qall[s] != qall[0])
{
re->type = ere;
snew(re->q[ere], re->nrepl);
- for (s = 0; s < ms->nsim; s++)
+ for (s = 0; s < ms->numSimulations_; s++)
{
re->q[ere][s] = qall[s];
}
fprintf(fplog, "\nInitializing Replica Exchange\n");
- if (!isMultiSim(ms) || ms->nsim == 1)
+ if (!isMultiSim(ms) || ms->numSimulations_ == 1)
{
gmx_fatal(FARGS,
"Nothing to exchange with only one replica, maybe you forgot to set the "
snew(re, 1);
- re->repl = ms->sim;
- re->nrepl = ms->nsim;
+ re->repl = ms->simulationIndex_;
+ re->nrepl = ms->numSimulations_;
snew(re->q, ereENDSINGLE);
fprintf(fplog, "Repl There are %d replicas:\n", re->nrepl);
/*
MPI_Sendrecv(v, n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
buf,n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
- ms->mpi_comm_masters,MPI_STATUS_IGNORE);
+ ms->mastersComm_,MPI_STATUS_IGNORE);
*/
{
MPI_Request mpi_req;
- MPI_Isend(v, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req);
- MPI_Recv(buf, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters,
- MPI_STATUS_IGNORE);
+ MPI_Isend(v, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req);
+ MPI_Recv(buf, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, MPI_STATUS_IGNORE);
MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
}
#endif
/*
MPI_Sendrecv(v, n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
buf,n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
- ms->mpi_comm_masters,MPI_STATUS_IGNORE);
+ ms->mastersComm_,MPI_STATUS_IGNORE);
*/
{
MPI_Request mpi_req;
- MPI_Isend(v, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req);
- MPI_Recv(buf, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters,
+ MPI_Isend(v, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req);
+ MPI_Recv(buf, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_,
MPI_STATUS_IGNORE);
MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
}
/*
MPI_Sendrecv(v[0], n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
buf[0],n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
- ms->mpi_comm_masters,MPI_STATUS_IGNORE);
+ ms->mastersComm_,MPI_STATUS_IGNORE);
*/
{
MPI_Request mpi_req;
- MPI_Isend(v[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req);
- MPI_Recv(buf[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters,
+ MPI_Isend(v[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req);
+ MPI_Recv(buf[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_,
MPI_STATUS_IGNORE);
MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
}
physicalNodeComm = PhysicalNodeCommunicator(communicator, gmx_physicalnode_id_hash());
}
- GMX_RELEASE_ASSERT(communicator == MPI_COMM_WORLD, "Must have valid world communicator");
- CommrecHandle crHandle = init_commrec(communicator, ms);
+ GMX_RELEASE_ASSERT(ms || communicator == MPI_COMM_WORLD,
+ "Must have valid world communicator unless running a multi-simulation");
+ CommrecHandle crHandle = init_commrec(communicator);
t_commrec* cr = crHandle.get();
GMX_RELEASE_ASSERT(cr != nullptr, "Must have valid commrec");
.appendTextFormatted(
"This is simulation %d out of %d running as a composite GROMACS\n"
"multi-simulation job. Setup for this simulation:\n",
- ms->sim, ms->nsim);
+ ms->simulationIndex_, ms->numSimulations_);
}
GMX_LOG(mdlog.warning)
.appendTextFormatted("Using %d MPI %s\n", cr->nnodes,
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
"Without it, the communicator must be null.");
if (!multiSimDirectoryNames.empty())
{
- multiSimulation_ = std::make_unique<gmx_multisim_t>(communicator, multiSimDirectoryNames);
+ multiSimulation_ = buildMultiSimulation(communicator, multiSimDirectoryNames);
+ // Use the communicator resulting from the split for the multi-simulation.
+ communicator_ = multiSimulation_->simulationComm_;
}
}
To help you identify which directories need attention, the %d
simulations wanted the following respective behaviors:
)",
- ms->nsim);
+ ms->numSimulations_);
for (index simIndex = 0; simIndex != ssize(startingBehaviors); ++simIndex)
{
auto behavior = static_cast<StartingBehavior>(startingBehaviors[simIndex]);
simulation checkpoint files were from the following respective
simulation parts:
)",
- ms->nsim);
+ ms->numSimulations_);
for (index partIndex = 0; partIndex != ssize(simulationParts); ++partIndex)
{
message += formatString(" Simulation %6zd: %d\n", partIndex, simulationParts[partIndex]);
#include "config.h"
#include "gromacs/mdtypes/commrec.h"
+#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/fatalerror.h"
#include "gromacs/utility/futil.h"
#include "gromacs/utility/gmxassert.h"
-#include "gromacs/utility/mpiinplacebuffers.h"
#include "gromacs/utility/smalloc.h"
-gmx_multisim_t::gmx_multisim_t() = default;
-
-gmx_multisim_t::gmx_multisim_t(MPI_Comm comm, gmx::ArrayRef<const std::string> multidirs)
+std::unique_ptr<gmx_multisim_t> buildMultiSimulation(MPI_Comm worldComm,
+ gmx::ArrayRef<const std::string> multidirs)
{
if (multidirs.empty())
{
- return;
+ return nullptr;
}
if (!GMX_LIB_MPI && !multidirs.empty())
{
- gmx_fatal(FARGS,
- "mdrun -multidir is only supported when GROMACS has been "
- "configured with a proper external MPI library.");
+ GMX_THROW(gmx::NotImplementedError(
+ "Multi-simulations are only supported when GROMACS has been "
+ "configured with a proper external MPI library."));
}
if (multidirs.size() == 1)
{
/* NOTE: It would be nice if this special case worked, but this requires checks/tests. */
- gmx_fatal(FARGS,
- "To run mdrun in multiple simulation mode, more then one "
- "actual simulation is required. The single simulation case is not supported.");
+ GMX_THROW(gmx::NotImplementedError(
+ "To run mdrun in multi-simulation mode, more then one "
+ "actual simulation is required. The single simulation case is not supported."));
}
-#if GMX_MPI
+#if GMX_LIB_MPI
int numRanks;
- MPI_Comm_size(comm, &numRanks);
+ MPI_Comm_size(worldComm, &numRanks);
if (numRanks % multidirs.size() != 0)
{
- gmx_fatal(FARGS,
- "The number of ranks (%d) is not a multiple of the number of simulations (%td)",
- numRanks, multidirs.ssize());
+ auto message = gmx::formatString(
+ "The number of ranks (%d) is not a multiple of the number of simulations (%td)",
+ numRanks, multidirs.ssize());
+ GMX_THROW(gmx::InconsistentInputError(message));
}
- int numRanksPerSim = numRanks / multidirs.size();
- int rankWithinComm;
- MPI_Comm_rank(comm, &rankWithinComm);
+ int numRanksPerSimulation = numRanks / multidirs.size();
+ int rankWithinWorldComm;
+ MPI_Comm_rank(worldComm, &rankWithinWorldComm);
if (debug)
{
fprintf(debug, "We have %td simulations, %d ranks per simulation, local simulation is %d\n",
- multidirs.ssize(), numRanksPerSim, rankWithinComm / numRanksPerSim);
+ multidirs.ssize(), numRanksPerSimulation, rankWithinWorldComm / numRanksPerSimulation);
}
- nsim = multidirs.size();
- sim = rankWithinComm / numRanksPerSim;
- /* Create a communicator for the master nodes */
- std::vector<int> rank(nsim);
- for (int i = 0; i < nsim; i++)
+ int numSimulations = multidirs.size();
+ // Create a communicator for the master ranks of each simulation
+ std::vector<int> ranksOfMasters(numSimulations);
+ for (int i = 0; i < numSimulations; i++)
{
- rank[i] = i * numRanksPerSim;
+ ranksOfMasters[i] = i * numRanksPerSimulation;
+ }
+ MPI_Group worldGroup;
+ // No need to free worldGroup later, we didn't create it.
+ MPI_Comm_group(worldComm, &worldGroup);
+
+ MPI_Group mastersGroup = MPI_GROUP_NULL;
+ MPI_Group_incl(worldGroup, numSimulations, ranksOfMasters.data(), &mastersGroup);
+ MPI_Comm mastersComm = MPI_COMM_NULL;
+ MPI_Comm_create(worldComm, mastersGroup, &mastersComm);
+ if (mastersGroup != MPI_GROUP_NULL)
+ {
+ MPI_Group_free(&mastersGroup);
}
- MPI_Group mpi_group_world;
- MPI_Comm_group(comm, &mpi_group_world);
- MPI_Group_incl(mpi_group_world, nsim, rank.data(), &mpi_group_masters);
- MPI_Comm_create(comm, mpi_group_masters, &mpi_comm_masters);
-
-# if !MPI_IN_PLACE_EXISTS
- /* initialize the MPI_IN_PLACE replacement buffers */
- snew(mpb, 1);
- mpb->ibuf = nullptr;
- mpb->libuf = nullptr;
- mpb->fbuf = nullptr;
- mpb->dbuf = nullptr;
- mpb->ibuf_alloc = 0;
- mpb->libuf_alloc = 0;
- mpb->fbuf_alloc = 0;
- mpb->dbuf_alloc = 0;
-# endif
- // TODO This should throw upon error
- gmx_chdir(multidirs[sim].c_str());
+ int simulationIndex = rankWithinWorldComm / numRanksPerSimulation;
+ MPI_Comm simulationComm = MPI_COMM_NULL;
+ MPI_Comm_split(worldComm, simulationIndex, rankWithinWorldComm, &simulationComm);
+
+ try
+ {
+ gmx_chdir(multidirs[simulationIndex].c_str());
+ }
+ catch (gmx::GromacsException& e)
+ {
+ e.prependContext("While changing directory for multi-simulation to " + multidirs[simulationIndex]);
+ throw;
+ }
+ return std::make_unique<gmx_multisim_t>(numSimulations, simulationIndex, mastersComm, simulationComm);
#else
- GMX_UNUSED_VALUE(comm);
+ GMX_UNUSED_VALUE(worldComm);
+ return nullptr;
#endif
}
-gmx_multisim_t::~gmx_multisim_t()
+gmx_multisim_t::gmx_multisim_t(int numSimulations, int simulationIndex, MPI_Comm mastersComm, MPI_Comm simulationComm) :
+ numSimulations_(numSimulations),
+ simulationIndex_(simulationIndex),
+ mastersComm_(mastersComm),
+ simulationComm_(simulationComm)
{
- done_mpi_in_place_buf(mpb);
+}
-#if GMX_MPI
+gmx_multisim_t::~gmx_multisim_t()
+{
+#if GMX_LIB_MPI
// TODO This would work better if the result of MPI_Comm_split was
// put into an RAII-style guard, such as gmx::unique_cptr.
- if (mpi_comm_masters != MPI_COMM_NULL && mpi_comm_masters != MPI_COMM_WORLD)
+ if (mastersComm_ != MPI_COMM_NULL && mastersComm_ != MPI_COMM_WORLD)
{
- MPI_Comm_free(&mpi_comm_masters);
+ MPI_Comm_free(&mastersComm_);
}
- if (mpi_group_masters != MPI_GROUP_NULL)
+ if (simulationComm_ != MPI_COMM_NULL && simulationComm_ != MPI_COMM_WORLD)
{
- MPI_Group_free(&mpi_group_masters);
+ MPI_Comm_free(&simulationComm_);
}
#endif
}
#if !GMX_MPI
GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumd_sim");
#else
- gmx_sumd_comm(nr, r, ms->mpi_comm_masters);
+ gmx_sumd_comm(nr, r, ms->mastersComm_);
#endif
}
#if !GMX_MPI
GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumf_sim");
#else
- gmx_sumf_comm(nr, r, ms->mpi_comm_masters);
+ gmx_sumf_comm(nr, r, ms->mastersComm_);
#endif
}
GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumi_sim");
#else
# if MPI_IN_PLACE_EXISTS
- MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mastersComm_);
# else
/* this is thread-unsafe, but it will do for now: */
- int i;
-
- if (nr > ms->mpb->ibuf_alloc)
- {
- ms->mpb->ibuf_alloc = nr;
- srenew(ms->mpb->ibuf, ms->mpb->ibuf_alloc);
- }
- MPI_Allreduce(r, ms->mpb->ibuf, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
- for (i = 0; i < nr; i++)
- {
- r[i] = ms->mpb->ibuf[i];
- }
+ ms->intBuffer.resize(nr);
+ MPI_Allreduce(r, ms->intBuffer.data(), ms->intBuffer.size(), MPI_INT, MPI_SUM, ms->mastersComm_);
+ std::copy(std::begin(ms->intBuffer), std::end(ms->intBuffer), r);
# endif
#endif
}
GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumli_sim");
#else
# if MPI_IN_PLACE_EXISTS
- MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT64_T, MPI_SUM, ms->mpi_comm_masters);
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT64_T, MPI_SUM, ms->mastersComm_);
# else
/* this is thread-unsafe, but it will do for now: */
- int i;
-
- if (nr > ms->mpb->libuf_alloc)
- {
- ms->mpb->libuf_alloc = nr;
- srenew(ms->mpb->libuf, ms->mpb->libuf_alloc);
- }
- MPI_Allreduce(r, ms->mpb->libuf, nr, MPI_INT64_T, MPI_SUM, ms->mpi_comm_masters);
- for (i = 0; i < nr; i++)
- {
- r[i] = ms->mpb->libuf[i];
- }
+ ms->int64Buffer.resize(nr);
+ MPI_Allreduce(r, ms->int64Buffer.data(), ms->int64Buffer.size(), MPI_INT64_T, MPI_SUM, ms->mastersComm_);
+ std::copy(std::begin(ms->int64Buffer), std::end(ms->int64Buffer), r);
# endif
#endif
}
#if GMX_MPI
if (ms != nullptr)
{
- valuesFromAllRanks.resize(ms->nsim);
- valuesFromAllRanks[ms->sim] = localValue;
- gmx_sumi_sim(ms->nsim, valuesFromAllRanks.data(), ms);
+ valuesFromAllRanks.resize(ms->numSimulations_);
+ valuesFromAllRanks[ms->simulationIndex_] = localValue;
+ gmx_sumi_sim(ms->numSimulations_, valuesFromAllRanks.data(), ms);
}
else
{
gmx_fatal(FARGS, "check_multi_int called with a NULL communication pointer");
}
- snew(ibuf, ms->nsim);
- ibuf[ms->sim] = val;
- gmx_sumi_sim(ms->nsim, ibuf, ms);
+ snew(ibuf, ms->numSimulations_);
+ ibuf[ms->simulationIndex_] = val;
+ gmx_sumi_sim(ms->numSimulations_, ibuf, ms);
bCompatible = TRUE;
- for (p = 1; p < ms->nsim; p++)
+ for (p = 1; p < ms->numSimulations_; p++)
{
bCompatible = bCompatible && (ibuf[p - 1] == ibuf[p]);
}
if (nullptr != log)
{
fprintf(log, "\n%s is not equal for all subsystems\n", name);
- for (p = 0; p < ms->nsim; p++)
+ for (p = 0; p < ms->numSimulations_; p++)
{
fprintf(log, " subsystem %d: %d\n", p, ibuf[p]);
}
}
- gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim);
+ gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->numSimulations_);
}
sfree(ibuf);
gmx_fatal(FARGS, "check_multi_int called with a NULL communication pointer");
}
- snew(ibuf, ms->nsim);
- ibuf[ms->sim] = val;
- gmx_sumli_sim(ms->nsim, ibuf, ms);
+ snew(ibuf, ms->numSimulations_);
+ ibuf[ms->simulationIndex_] = val;
+ gmx_sumli_sim(ms->numSimulations_, ibuf, ms);
bCompatible = TRUE;
- for (p = 1; p < ms->nsim; p++)
+ for (p = 1; p < ms->numSimulations_; p++)
{
bCompatible = bCompatible && (ibuf[p - 1] == ibuf[p]);
}
if (nullptr != log)
{
fprintf(log, "\n%s is not equal for all subsystems\n", name);
- for (p = 0; p < ms->nsim; p++)
+ for (p = 0; p < ms->numSimulations_; p++)
{
char strbuf[255];
/* first make the format string */
fprintf(log, strbuf, p, ibuf[p]);
}
}
- gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim);
+ gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->numSimulations_);
}
sfree(ibuf);
// Ranks of multi-simulations know whether they are a master
// rank. Ranks of non-multi simulation do not know until a
// t_commrec is available.
- if ((ms != nullptr) && (ms->nsim > 1))
+ if ((ms != nullptr) && (ms->numSimulations_ > 1))
{
- return ms->mpi_comm_masters != MPI_COMM_NULL;
+ return ms->mastersComm_ != MPI_COMM_NULL;
}
else
{
bool isMasterSim(const gmx_multisim_t* ms)
{
- return !isMultiSim(ms) || ms->sim == 0;
+ return !isMultiSim(ms) || ms->simulationIndex_ == 0;
}
bool isMasterSimMasterRank(const gmx_multisim_t* ms, const bool isMaster)
#include <memory>
#include <string>
+#include <vector>
#include "gromacs/utility/arrayref.h"
#include "gromacs/utility/gmxmpi.h"
-#include "gromacs/utility/mpiinplacebuffers.h"
+
+struct gmx_multisim_t;
+
+/*! \libinternal
+ * \brief Builder function for gmx_multisim_t
+ *
+ * \param[in] worldComm MPI communicator to split when
+ * multi-simulation is requested.
+ * \param[in] multidirs Strings naming the subdirectories when
+ * multi-simulation is requested, otherwise empty
+ *
+ * Splits \c worldComm into \c multidirs.size() separate
+ * simulations, if >1, and creates a communication structure
+ * between the master ranks of these simulations.
+ *
+ * Valid to call regardless of build configuration, but \c
+ * multidirs must be empty unless a real MPI build is used.
+ *
+ * \throws NotImplementedError when \c multidirs is non-empty unless using real MPI is true
+ * \throws NotImplementedError when \c multidirs has exactly one element
+ * \throws InconsistentInputError when the number of MPI ranks is not a multiple of the number of \c multidirs
+ * \throws FileIOError when the simulation cannot change to the working directory in \c multidirs
+ */
+std::unique_ptr<gmx_multisim_t> buildMultiSimulation(MPI_Comm worldComm,
+ gmx::ArrayRef<const std::string> multidirs);
/*! \libinternal
* \brief Coordinate multi-simulation resources for mdrun
*/
struct gmx_multisim_t
{
- //! Default constructor
- gmx_multisim_t();
- /*! \brief Constructor useful for mdrun simulations
+ /*! \brief Constructor
*
- * Splits the communicator into multidirs.size() separate
- * simulations, if >1, and creates a communication structure
- * between the master these simulations.
+ * \param[in] numSimulations The number of simulations in the MPI world.
+ * \param[in] simulationIndex The index of this simulation in the set of simulations.
+ * \param[in] mastersComm On master ranks, the communicator among master ranks;
+ * otherwise MPI_COMM_NULL.
+ * \param[in] simulationComm The communicator among ranks of this simulation.
*
- * Valid to call regardless of build configuration, but \c
- * multidirs must be empty unless a real MPI build is used. */
- gmx_multisim_t(MPI_Comm comm, gmx::ArrayRef<const std::string> multidirs);
+ * Assumes ownership of the communicators if they are neither
+ * MPI_COMM_WORLD nor MPI_COMM_NULL. If so, upon destruction will
+ * call MPI_Comm_free on them.
+ */
+ gmx_multisim_t(int numSimulations, int simulationIndex, MPI_Comm mastersComm, MPI_Comm simulationComm);
//! Destructor
~gmx_multisim_t();
//! The number of simulations in the set of multi-simulations
- int nsim = 1;
+ int numSimulations_ = 1;
//! The index of the simulation that owns this object within the set
- int sim = 0;
- //! The MPI Group between master ranks of simulations, valid only on master ranks.
- MPI_Group mpi_group_masters = MPI_GROUP_NULL;
+ int simulationIndex_ = 0;
//! The MPI communicator between master ranks of simulations, valid only on master ranks.
- MPI_Comm mpi_comm_masters = MPI_COMM_NULL;
- //! Communication buffers needed if MPI_IN_PLACE isn't supported
- mpi_in_place_buf_t* mpb = nullptr;
+ MPI_Comm mastersComm_ = MPI_COMM_NULL;
+ //! The MPI communicator between ranks of this simulation.
+ MPI_Comm simulationComm_ = MPI_COMM_NULL;
+ /*! \brief Communication buffers needed if MPI_IN_PLACE isn't supported
+ *
+ * Other types could be added as needed.
+ *
+ * These vectors are unused when MPI_IN_PLACE is available
+ * and could be removed with preprocessing (or perhaps
+ * templating) or simply requiring MPI 2.0 (the standard
+ * introduced in 1997). However, the additional cache pressure
+ * introduced by the extra size of this type is not of great
+ * concern, since we have at most one per MPI rank.
+ * See issue #3591. */
+ //! \{
+ std::vector<int> intBuffer_;
+ std::vector<int64_t> int64Buffer_;
+ //! \}
};
//! Calculate the sum over the simulations of an array of ints
* all detected ncore_tot physical cores. We are currently not
* checking for that here.
*/
- int numRanksTot = cr->nnodes * (isMultiSim(ms) ? ms->nsim : 1);
+ int numRanksTot = cr->nnodes * (isMultiSim(ms) ? ms->numSimulations_ : 1);
int numAtomsPerRank = mtop.natoms / cr->nnodes;
int numCoresPerRank = hwinfo.ncore_tot / numRanksTot;
if (numAtomsPerRank < c_numAtomsPerCoreSquaredSmtThreshold * gmx::square(numCoresPerRank))
#define NFILE asize(fnm)
- CommrecHandle commrecHandle = init_commrec(MPI_COMM_WORLD, nullptr);
+ CommrecHandle commrecHandle = init_commrec(MPI_COMM_WORLD);
t_commrec* cr = commrecHandle.get();
PCA_Flags = PCA_NOEXIT_ON_ARGS;
#endif
if (rc != 0)
{
- gmx_fatal(FARGS, "Cannot change directory to '%s'. Reason: %s", directory, strerror(errno));
+ auto message = gmx::formatString("Cannot change directory to '%s'. Reason: %s", directory,
+ strerror(errno));
+ GMX_THROW(gmx::FileIOError(message));
}
}
+++ /dev/null
-/*
- * This file is part of the GROMACS molecular simulation package.
- *
- * Copyright (c) 2019, by the GROMACS development team, led by
- * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
- * and including many others, as listed in the AUTHORS file in the
- * top-level source directory and at http://www.gromacs.org.
- *
- * GROMACS is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2.1
- * of the License, or (at your option) any later version.
- *
- * GROMACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with GROMACS; if not, see
- * http://www.gnu.org/licenses, or write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * If you want to redistribute modifications to GROMACS, please
- * consider that scientific software is very special. Version
- * control is crucial - bugs must be traceable. We will be happy to
- * consider code for inclusion in the official distribution, but
- * derived work must not be called official GROMACS. Details are found
- * in the README & COPYING files - if they are missing, get the
- * official version at http://www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org.
- */
-/*! \internal \file
- * \brief Defines cleanup function for the copies necessary when
- * MPI_IN_PLACE is not supported by the MPI library.
- *
- * \author Mark Abraham <mark.j.abraham@gmail.com>
- * \ingroup module_utility
- */
-#include "gmxpre.h"
-
-#include "mpiinplacebuffers.h"
-
-#include "gromacs/utility/smalloc.h"
-
-void done_mpi_in_place_buf(mpi_in_place_buf_t* buf)
-{
- if (nullptr != buf)
- {
- sfree(buf->ibuf);
- sfree(buf->libuf);
- sfree(buf->fbuf);
- sfree(buf->dbuf);
- sfree(buf);
- }
-}
+++ /dev/null
-/*
- * This file is part of the GROMACS molecular simulation package.
- *
- * Copyright (c) 2019, by the GROMACS development team, led by
- * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
- * and including many others, as listed in the AUTHORS file in the
- * top-level source directory and at http://www.gromacs.org.
- *
- * GROMACS is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2.1
- * of the License, or (at your option) any later version.
- *
- * GROMACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with GROMACS; if not, see
- * http://www.gnu.org/licenses, or write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * If you want to redistribute modifications to GROMACS, please
- * consider that scientific software is very special. Version
- * control is crucial - bugs must be traceable. We will be happy to
- * consider code for inclusion in the official distribution, but
- * derived work must not be called official GROMACS. Details are found
- * in the README & COPYING files - if they are missing, get the
- * official version at http://www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org.
- */
-/*! \libinternal \file
- * \brief Declares object that provides buffers for the copies
- * necessary when MPI_IN_PLACE is not supported by the MPI library.
- *
- * The struct is declared in the header so that functionality it
- * supports can be inlined.
- *
- * \author Mark Abraham <mark.j.abraham@gmail.com>
- * \inlibraryapi
- * \ingroup module_utility
- */
-#ifndef GMX_UTILTIY_MPIINPLACEBUFFERS_H
-#define GMX_UTILTIY_MPIINPLACEBUFFERS_H
-
-#include <cstdint>
-
-struct mpi_in_place_buf_t
-{
- /* these buffers are used as destination buffers if MPI_IN_PLACE isn't
- supported.*/
- int* ibuf; /* for ints */
- int ibuf_alloc;
-
- int64_t* libuf;
- int libuf_alloc;
-
- float* fbuf; /* for floats */
- int fbuf_alloc;
-
- double* dbuf; /* for doubles */
- int dbuf_alloc;
-};
-
-//! Cleans up the buffers
-void done_mpi_in_place_buf(mpi_in_place_buf_t* buf);
-
-#endif
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff