Changed default value for nmultidir variable.
Called done_commrec.
Change-Id: Ie5f531389d682b9a619b5fc6314cb687f1a68828
GMX_RELEASE_ASSERT(static_cast<size_t>(bias.ndim()) == pullCoordIndex.size(), "The bias dimensionality should match the number of pull coordinates.");
}
-Awh::Awh(FILE *fplog,
- const t_inputrec &inputRecord,
- const t_commrec *commRecord,
- const AwhParams &awhParams,
- const std::string &biasInitFilename,
- pull_t *pull_work) :
+Awh::Awh(FILE *fplog,
+ const t_inputrec &inputRecord,
+ const t_commrec *commRecord,
+ const gmx_multisim_t *multiSimRecord,
+ const AwhParams &awhParams,
+ const std::string &biasInitFilename,
+ pull_t *pull_work) :
seed_(awhParams.seed),
nstout_(awhParams.nstOut),
commRecord_(commRecord),
+ multiSimRecord_(multiSimRecord),
pull_(pull_work),
potentialOffset_(0)
{
}
int numSharingSimulations = 1;
- if (awhParams.shareBiasMultisim && isMultiSim(commRecord_->ms))
+ if (awhParams.shareBiasMultisim && isMultiSim(multiSimRecord_))
{
- numSharingSimulations = commRecord_->ms->nsim;
+ numSharingSimulations = multiSimRecord_->nsim;
}
/* Initialize all the biases */
pointSize.push_back(biasCts.bias.state().points().size());
}
/* Ensure that the shared biased are compatible between simulations */
- biasesAreCompatibleForSharingBetweenSimulations(awhParams, pointSize, commRecord_->ms);
+ biasesAreCompatibleForSharingBetweenSimulations(awhParams, pointSize, multiSimRecord_);
}
}
gmx::ArrayRef<const double> biasForce =
biasCts.bias.calcForceAndUpdateBias(coordValue,
&biasPotential, &biasPotentialJump,
- commRecord_->ms,
+ multiSimRecord_,
t, step, seed_, fplog);
awhPotential += biasPotential;
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* \param[in,out] fplog General output file, normally md.log, can be nullptr.
* \param[in] inputRecord General input parameters (as set up by grompp).
* \param[in] commRecord Struct for communication, can be nullptr.
+ * \param[in] multiSimRecord Multi-sim handler
* \param[in] awhParams AWH input parameters, consistent with the relevant parts of \p inputRecord (as set up by grompp).
* \param[in] biasInitFilename Name of file to read PMF and target from.
* \param[in,out] pull_work Pointer to a pull struct which AWH will couple to, has to be initialized, is assumed not to change during the lifetime of the Awh object.
*/
- Awh(FILE *fplog,
- const t_inputrec &inputRecord,
- const t_commrec *commRecord,
- const AwhParams &awhParams,
- const std::string &biasInitFilename,
- pull_t *pull_work);
+ Awh(FILE *fplog,
+ const t_inputrec &inputRecord,
+ const t_commrec *commRecord,
+ const gmx_multisim_t *multiSimRecord,
+ const AwhParams &awhParams,
+ const std::string &biasInitFilename,
+ pull_t *pull_work);
/*! \brief Destructor. */
~Awh();
const gmx_int64_t seed_; /**< Random seed for MC jumping with umbrella type bias potential. */
const int nstout_; /**< Interval in steps for writing to energy file. */
const t_commrec *commRecord_; /**< Pointer to the communication record. */
+ const gmx_multisim_t *multiSimRecord_; /**< Handler for multi-simulations. */
pull_t *pull_; /**< Pointer to the pull working data. */
double potentialOffset_; /**< The offset of the bias potential which changes due to bias updates. */
};
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
while (((i+n) < disres->nr) &&
(forceparams[forceatoms[i+n]].disres.label == label));
- calc_disres_R_6(nullptr, n, &forceatoms[i],
+ calc_disres_R_6(nullptr, nullptr, n, &forceatoms[i],
(const rvec*)x, pbc, fcd, nullptr);
if (fcd->disres.Rt_6[label] <= 0)
}
ir->dr_tau = 0.0;
- init_disres(fplog, &mtop, ir, nullptr, &fcd, nullptr, FALSE);
+ init_disres(fplog, &mtop, ir, nullptr, nullptr, &fcd, nullptr, FALSE);
natoms = read_first_x(oenv, &status, ftp2fn(efTRX, NFILE, fnm), &t, &x, box);
snew(f, 5*natoms);
/* The source code in this file should be thread-safe.
Please keep it that way. */
-void gmx_fill_commrec_from_mpi(t_commrec gmx_unused *cr)
+void gmx_fill_commrec_from_mpi(t_commrec *cr,
+ const gmx_multisim_t *ms)
{
#if !GMX_MPI
gmx_call("gmx_fill_commrec_from_mpi");
+ GMX_UNUSED_VALUE(cr);
+ GMX_UNUSED_VALUE(ms);
#else
if (!gmx_mpi_initialized())
{
// all multi-node MPI cases with more than one PP rank per node,
// with and without GPUs. By always having it available, we also
// don't need to protect calls to mpi_comm_physicalnode, etc.
- if (PAR(cr) || isMultiSim(cr->ms))
+ if (PAR(cr) || isMultiSim(ms))
{
MPI_Comm_split(MPI_COMM_WORLD, gmx_physicalnode_id_hash(), cr->nodeid, &cr->mpi_comm_physicalnode);
}
snew(cr, 1);
+ cr->mpi_comm_physicalnode = MPI_COMM_NULL;
#if GMX_LIB_MPI
- gmx_fill_commrec_from_mpi(cr);
+ gmx_fill_commrec_from_mpi(cr, nullptr);
#else
- cr->mpi_comm_mysim = nullptr;
- cr->mpi_comm_mygroup = nullptr;
+ cr->mpi_comm_mysim = MPI_COMM_NULL;
+ cr->mpi_comm_mygroup = MPI_COMM_NULL;
cr->nnodes = 1;
cr->sim_nodeid = 0;
cr->nodeid = cr->sim_nodeid;
return cr;
}
-static void done_mpi_in_place_buf(mpi_in_place_buf_t *buf)
+void done_mpi_in_place_buf(mpi_in_place_buf_t *buf)
{
if (nullptr != buf)
{
void done_commrec(t_commrec *cr)
{
#if GMX_MPI
- if (PAR(cr) || isMultiSim(cr->ms))
+ if (cr->mpi_comm_physicalnode != MPI_COMM_NULL)
{
MPI_Comm_free(&cr->mpi_comm_physicalnode);
}
// TODO: implement
// done_domdec(cr->dd);
}
- if (nullptr != cr->ms)
- {
- done_mpi_in_place_buf(cr->ms->mpb);
- sfree(cr->ms);
- }
done_mpi_in_place_buf(cr->mpb);
sfree(cr);
}
-t_commrec *reinitialize_commrec_for_this_thread(const t_commrec gmx_unused *cro)
+t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro,
+ const gmx_multisim_t *ms)
{
#if GMX_THREAD_MPI
t_commrec *cr;
*cr = *cro;
/* and we start setting our own thread-specific values for things */
- gmx_fill_commrec_from_mpi(cr);
+ gmx_fill_commrec_from_mpi(cr, ms);
// TODO cr->duty should not be initialized here
cr->duty = (DUTY_PP | DUTY_PME);
return cr;
#else
+ GMX_UNUSED_VALUE(cro);
+ GMX_UNUSED_VALUE(ms);
return nullptr;
#endif
}
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
struct t_commrec *init_commrec(void);
/* Allocate, initialize and return the commrec. */
-void done_commrec(struct t_commrec *cr);
+void done_commrec(t_commrec *cr);
/* Free memory associated with the commrec. */
-struct t_commrec *reinitialize_commrec_for_this_thread(const struct t_commrec *cro);
+struct t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro,
+ const gmx_multisim_t *ms);
+
/* Initialize communication records for thread-parallel simulations.
Must be called on all threads before any communication takes place by
the individual threads. Copies the original commrec to
thread-local versions (a small memory leak results because we don't
deallocate the old shared version). */
-void gmx_fill_commrec_from_mpi(struct t_commrec *cr);
+void gmx_fill_commrec_from_mpi(t_commrec *cr,
+ const gmx_multisim_t *ms);
/* Continues t_commrec construction */
void gmx_setup_nodecomm(FILE *fplog, struct t_commrec *cr);
}
void gmx_print_detected_hardware(FILE *fplog, const t_commrec *cr,
+ const gmx_multisim_t *ms,
const gmx::MDLogger &mdlog,
const gmx_hw_info_t *hwinfo)
{
*/
if (cpuInfo.supportLevel() >= gmx::CpuInfo::SupportLevel::Features)
{
- gmx::simdCheck(static_cast<gmx::SimdType>(hwinfo->simd_suggest_min), fplog, isMasterSimMasterRank(cr->ms, cr));
+ gmx::simdCheck(static_cast<gmx::SimdType>(hwinfo->simd_suggest_min), fplog, isMasterSimMasterRank(ms, cr));
}
/* For RDTSCP we only check on our local node and skip the MPI reduction */
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include <cstdio>
struct gmx_hw_info_t;
+struct gmx_multisim_t;
struct t_commrec;
namespace gmx
* and to stderr the master rank.
*/
void gmx_print_detected_hardware(FILE *fplog, const t_commrec *cr,
+ const gmx_multisim_t *ms,
const gmx::MDLogger &mdlog,
const gmx_hw_info_t *hwinfo);
void init_IMD(t_inputrec *ir,
t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx_mtop_t *top_global,
FILE *fplog,
int defnstimd,
if (options.wait || options.terminatable || options.pull)
{
/* Multiple simulations or replica exchange */
- if (isMultiSim(cr->ms))
+ if (isMultiSim(ms))
{
fprintf(stderr, "%s Cannot use IMD for multiple simulations or replica exchange.\n", IMDstr);
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
struct gmx_domdec_t;
struct gmx_enerdata_t;
struct gmx_mtop_t;
+struct gmx_multisim_t;
struct gmx_output_env_t;
struct gmx_wallcycle;
struct MdrunOptions;
* \param ir The inputrec structure containing the MD input parameters
* including a pointer to the IMD data structure.
* \param cr Information structure for MPI communication.
+ * \param ms Handler for multi-simulations.
* \param top_global The topology of the whole system.
* \param fplog General output file, normally md.log.
* \param defnstimd Default IMD update (=communication) frequency.
* \param oenv Output options.
* \param mdrunOptions Options for mdrun.
*/
-void init_IMD(t_inputrec *ir, t_commrec *cr, gmx_mtop_t *top_global,
+void init_IMD(t_inputrec *ir, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx_mtop_t *top_global,
FILE *fplog, int defnstimd, rvec x[],
int nfile, const t_filenm fnm[], const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions);
void init_disres(FILE *fplog, const gmx_mtop_t *mtop,
t_inputrec *ir, const t_commrec *cr,
+ const gmx_multisim_t *ms,
t_fcdata *fcd, t_state *state, gmx_bool bIsREMD)
{
int fa, nmol, npair, np;
dd->Rtav_6 = &(dd->Rt_6[dd->nres]);
ptr = getenv("GMX_DISRE_ENSEMBLE_SIZE");
- if (cr && cr->ms != nullptr && ptr != nullptr && !bIsREMD)
+ if (cr && ms != nullptr && ptr != nullptr && !bIsREMD)
{
#if GMX_MPI
dd->nsystems = 0;
* than one processor per simulation system. */
if (MASTER(cr))
{
- check_multi_int(fplog, cr->ms, dd->nsystems,
+ check_multi_int(fplog, ms, dd->nsystems,
"the number of systems per ensemble",
FALSE);
}
* of ms->nsim. But this required an extra communicator which
* was stored in t_fcdata. This pulled in mpi.h in nearly all C files.
*/
- if (!(cr->ms->nsim == 1 || cr->ms->nsim == dd->nsystems))
+ if (!(ms->nsim == 1 || ms->nsim == dd->nsystems))
{
- gmx_fatal(FARGS, "GMX_DISRE_ENSEMBLE_SIZE (%d) is not equal to 1 or the number of systems (option -multidir) %d", dd->nsystems, cr->ms->nsim);
+ gmx_fatal(FARGS, "GMX_DISRE_ENSEMBLE_SIZE (%d) is not equal to 1 or the number of systems (option -multidir) %d", dd->nsystems, ms->nsim);
}
if (fplog)
{
for (int i = 0; i < dd->nsystems; i++)
{
fprintf(fplog, " %d",
- (cr->ms->sim/dd->nsystems)*dd->nsystems+i);
+ (ms->sim/dd->nsystems)*dd->nsystems+i);
}
fprintf(fplog, "\n");
}
* checks from appropriate processes (since check_multi_int is
* too broken to check whether the communication will
* succeed...) */
- if (cr && cr->ms && dd->nsystems > 1 && MASTER(cr))
+ if (cr && ms && dd->nsystems > 1 && MASTER(cr))
{
- check_multi_int(fplog, cr->ms, fcd->disres.nres,
+ check_multi_int(fplog, ms, fcd->disres.nres,
"the number of distance restraints",
FALSE);
}
}
void calc_disres_R_6(const t_commrec *cr,
+ const gmx_multisim_t *ms,
int nfa, const t_iatom forceatoms[],
const rvec x[], const t_pbc *pbc,
t_fcdata *fcd, history_t *hist)
Rtav_6[res] *= invn;
}
- GMX_ASSERT(cr != NULL && cr->ms != NULL, "We need multisim with nsystems>1");
- gmx_sum_sim(2*dd->nres, dd->Rt_6, cr->ms);
+ GMX_ASSERT(cr != NULL && ms != NULL, "We need multisim with nsystems>1");
+ gmx_sum_sim(2*dd->nres, dd->Rt_6, ms);
if (DOMAINDECOMP(cr))
{
#include "gromacs/utility/basedefinitions.h"
struct gmx_mtop_t;
+struct gmx_multisim_t;
class history_t;
struct t_commrec;
struct t_inputrec;
*/
void init_disres(FILE *fplog, const gmx_mtop_t *mtop,
t_inputrec *ir, const t_commrec *cr,
+ const gmx_multisim_t *ms,
t_fcdata *fcd, t_state *state, gmx_bool bIsREMD);
/*! \brief
* and the ensemble averaged r^-6 (inst. and time averaged) for all restraints
*/
void calc_disres_R_6(const t_commrec *cr,
+ const gmx_multisim_t *ms,
int nfa, const t_iatom *fa,
const rvec *x, const t_pbc *pbc,
t_fcdata *fcd, history_t *hist);
}
void calc_listed(const t_commrec *cr,
+ const gmx_multisim_t *ms,
struct gmx_wallcycle *wcycle,
const t_idef *idef,
const rvec x[], history_t *hist,
*/
GMX_RELEASE_ASSERT(fr->ePBC == epbcNONE || g != nullptr, "With orientation restraints molecules should be whole");
enerd->term[F_ORIRESDEV] =
- calc_orires_dev(cr->ms, idef->il[F_ORIRES].nr,
+ calc_orires_dev(ms, idef->il[F_ORIRES].nr,
idef->il[F_ORIRES].iatoms,
idef->iparams, md, x,
pbc_null, fcd, hist);
}
if (fcd->disres.nres > 0)
{
- calc_disres_R_6(cr,
+ calc_disres_R_6(cr, ms,
idef->il[F_DISRES].nr,
idef->il[F_DISRES].iatoms,
x, pbc_null,
matrix box,
const t_lambda *fepvals,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const t_idef *idef,
const rvec x[],
history_t *hist,
/* Not enough flops to bother counting */
set_pbc(&pbc_full, fr->ePBC, box);
}
- calc_listed(cr, wcycle, idef, x, hist,
+ calc_listed(cr, ms, wcycle, idef, x, hist,
forceForUseWithShiftForces, forceWithVirial,
fr, pbc, &pbc_full,
graph, enerd, nrnb, lambda, md, fcd,
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
struct gmx_enerdata_t;
struct gmx_grppairener_t;
+struct gmx_multisim_t;
class history_t;
struct t_commrec;
struct t_fcdata;
* Note that pbc_full is used only for position restraints, and is
* not initialized if there are none. */
void calc_listed(const t_commrec *cr,
+ const gmx_multisim_t *ms,
struct gmx_wallcycle *wcycle,
const t_idef *idef,
const rvec x[], history_t *hist,
matrix box,
const t_lambda *fepvals,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const t_idef *idef,
const rvec x[],
history_t *hist,
// TODO This implementation of ensemble orientation restraints is nasty because
// a user can't just do multi-sim with single-sim orientation restraints.
-void init_orires(FILE *fplog,
- const gmx_mtop_t *mtop,
- const t_inputrec *ir,
- const t_commrec *cr,
- t_state *globalState,
- t_oriresdata *od)
+void init_orires(FILE *fplog,
+ const gmx_mtop_t *mtop,
+ const t_inputrec *ir,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ t_state *globalState,
+ t_oriresdata *od)
{
od->nr = gmx_mtop_ftype_count(mtop, F_ORIRES);
if (0 == od->nr)
*/
snew(od->Dinsl, od->nr);
- const gmx_multisim_t *ms = cr->ms;
if (ms)
{
snew(od->Dins, od->nr);
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2010,2014,2015,2017, by the GROMACS development team, led by
+ * Copyright (c) 2010,2014,2015,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* on the master rank (which is the only rank, since orientation
* restraints can not run in parallel).
*/
-void init_orires(FILE *fplog,
- const gmx_mtop_t *mtop,
- const t_inputrec *ir,
- const t_commrec *cr,
- t_state *globalState,
- t_oriresdata *od);
+void init_orires(FILE *fplog,
+ const gmx_mtop_t *mtop,
+ const t_inputrec *ir,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ t_state *globalState,
+ t_oriresdata *od);
/*! \brief
* Calculates the time averaged D matrices, the S matrix for each experiment.
gmx_int64_t step,
struct gmx_lincsdata *lincsd, t_mdatoms *md,
t_commrec *cr,
+ const gmx_multisim_t *ms,
rvec *x, rvec *xprime, rvec *min_proj,
matrix box, t_pbc *pbc,
real lambda, real *dvdlambda,
{
cconerr(lincsd, xprime, pbc,
&ncons_loc, &p_ssd, &p_max, &p_imax);
- if (isMultiSim(cr->ms))
+ if (isMultiSim(ms))
{
- sprintf(buf3, " in simulation %d", cr->ms->sim);
+ sprintf(buf3, " in simulation %d", ms->sim);
}
else
{
struct gmx_constr *constr,
t_idef *idef, t_inputrec *ir,
t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx_int64_t step, int delta_step,
real step_scaling,
t_mdatoms *md,
if (constr->lincsd != nullptr)
{
- bOK = constrain_lincs(fplog, bLog, bEner, ir, step, constr->lincsd, md, cr,
+ bOK = constrain_lincs(fplog, bLog, bEner, ir, step, constr->lincsd, md, cr, ms,
x, xprime, min_proj,
box, pbc_null, lambda, dvdlambda,
invdt, v, vir != nullptr, vir_r_m_dr,
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "gromacs/topology/topology.h"
#include "gromacs/utility/real.h"
+struct gmx_multisim_t;
struct t_inputrec;
/* Abstract type for LINCS that is defined only in the file that uses it */
gmx_constr_t constr,
t_idef *idef,
t_inputrec *ir,
- struct t_commrec *cr,
+ t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx_int64_t step, int delta_step,
real step_scaling,
t_mdatoms *md,
gmx_int64_t step,
gmx_lincsdata_t lincsd, t_mdatoms *md,
struct t_commrec *cr,
+ const gmx_multisim_t *ms,
rvec *x, rvec *xprime, rvec *min_proj,
matrix box, struct t_pbc *pbc,
real lambda, real *dvdlambda,
void do_force_lowlevel(t_forcerec *fr, t_inputrec *ir,
t_idef *idef, t_commrec *cr,
+ const gmx_multisim_t *ms,
t_nrnb *nrnb, gmx_wallcycle_t wcycle,
t_mdatoms *md,
rvec x[], history_t *hist,
TRUE, box);
}
- do_force_listed(wcycle, box, ir->fepvals, cr,
+ do_force_listed(wcycle, box, ir->fepvals, cr, ms,
idef, (const rvec *) x, hist,
forceForUseWithShiftForces, forceWithVirial,
fr, &pbc, graph, enerd, nrnb, lambda, md, fcd,
struct gmx_edsam;
struct gmx_gpu_info_t;
struct gmx_groups_t;
+struct gmx_multisim_t;
struct gmx_vsite_t;
class history_t;
struct nonbonded_verlet_t;
const gmx_mtop_t *mtop);
void do_force(FILE *log, t_commrec *cr,
+ const gmx_multisim_t *ms,
t_inputrec *inputrec,
gmx_int64_t step, struct t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_localtop_t *top,
t_inputrec *ir,
t_idef *idef,
t_commrec *cr,
+ const gmx_multisim_t *ms,
t_nrnb *nrnb,
gmx_wallcycle_t wcycle,
t_mdatoms *md,
/* Call all the force routines */
void free_gpu_resources(const t_forcerec *fr,
- const t_commrec *cr);
+ const t_commrec *cr,
+ const gmx_multisim_t *ms);
#endif
* that it's not needed anymore (with a shared GPU run).
*/
void free_gpu_resources(const t_forcerec *fr,
- const t_commrec *cr)
+ const t_commrec *cr,
+ const gmx_multisim_t *ms)
{
bool isPPrankUsingGPU = fr && fr->nbv && fr->nbv->bUseGPU;
* Note: it is safe to not call the barrier on the ranks which do not use GPU,
* but it is easier and more futureproof to call it on the whole node.
*/
- if (GMX_THREAD_MPI && (PAR(cr) || isMultiSim(cr->ms)))
+ if (GMX_THREAD_MPI && (PAR(cr) || isMultiSim(ms)))
{
gmx_barrier_physical_node(cr);
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
class energyhistory_t;
struct gmx_mtop_t;
struct gmx_membed_t;
+struct gmx_multisim_t;
struct gmx_output_env_t;
struct MdrunOptions;
struct ObservablesHistory;
*
* \param[in] fplog Log file for output
* \param[in] cr Communication record
+ * \param[in] ms Handle to multi-simulation handler.
* \param[in] mdlog Log writer for important output
* \param[in] nfile Number of files
* \param[in] fnm Filename structure array
* \param[in] membed Membrane embedding data structure
* \param[in] walltime_accounting More timing information
*/
-typedef double integrator_t (FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+typedef double integrator_t (FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
}
// TODO move this to multi-sim module
-void init_multisystem(t_commrec *cr, int nsim, char **multidirs)
+gmx_multisim_t *init_multisystem(MPI_Comm comm, int nsim, char **multidirs)
{
gmx_multisim_t *ms;
- int nnodes, nnodpersim, sim;
#if GMX_MPI
MPI_Group mpi_group_world;
int *rank;
#endif
- if (nsim < 1)
+ if (nsim <= 1)
{
- return;
+ return nullptr;
}
if (!GMX_LIB_MPI && nsim > 1)
{
}
GMX_RELEASE_ASSERT(multidirs, "Must have multiple directories for -multisim");
- nnodes = cr->nnodes;
- if (nnodes % nsim != 0)
+#if GMX_MPI
+ int numRanks;
+ MPI_Comm_size(comm, &numRanks);
+ if (numRanks % nsim != 0)
{
- gmx_fatal(FARGS, "The number of ranks (%d) is not a multiple of the number of simulations (%d)", nnodes, nsim);
+ gmx_fatal(FARGS, "The number of ranks (%d) is not a multiple of the number of simulations (%d)", numRanks, nsim);
}
- nnodpersim = nnodes/nsim;
- sim = cr->nodeid/nnodpersim;
+ int numRanksPerSim = numRanks/nsim;
+ int rankWithinComm;
+ MPI_Comm_rank(comm, &rankWithinComm);
if (debug)
{
- fprintf(debug, "We have %d simulations, %d ranks per simulation, local simulation is %d\n", nsim, nnodpersim, sim);
+ fprintf(debug, "We have %d simulations, %d ranks per simulation, local simulation is %d\n", nsim, numRanksPerSim, rankWithinComm/numRanksPerSim);
}
- snew(ms, 1);
- cr->ms = ms;
+ ms = new gmx_multisim_t;
ms->nsim = nsim;
- ms->sim = sim;
-#if GMX_MPI
+ ms->sim = rankWithinComm/numRanksPerSim;
/* Create a communicator for the master nodes */
snew(rank, ms->nsim);
for (int i = 0; i < ms->nsim; i++)
{
- rank[i] = i*nnodpersim;
+ rank[i] = i*numRanksPerSim;
}
- MPI_Comm_group(MPI_COMM_WORLD, &mpi_group_world);
- MPI_Group_incl(mpi_group_world, nsim, rank, &ms->mpi_group_masters);
+ MPI_Comm_group(comm, &mpi_group_world);
+ MPI_Group_incl(mpi_group_world, ms->nsim, rank, &ms->mpi_group_masters);
sfree(rank);
MPI_Comm_create(MPI_COMM_WORLD, ms->mpi_group_masters,
&ms->mpi_comm_masters);
ms->mpb->dbuf_alloc = 0;
#endif
+ // TODO This should throw upon error
+ gmx_chdir(multidirs[ms->sim]);
+#else
+ GMX_UNUSED_VALUE(comm);
+ ms = nullptr;
#endif
- /* Reduce the intra-simulation communication */
- cr->sim_nodeid = cr->nodeid % nnodpersim;
- cr->nnodes = nnodpersim;
-#if GMX_MPI
- MPI_Comm_split(MPI_COMM_WORLD, sim, cr->sim_nodeid, &cr->mpi_comm_mysim);
- cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
- cr->nodeid = cr->sim_nodeid;
-#endif
+ return ms;
+}
- if (debug)
+void done_multisim(gmx_multisim_t *ms)
+{
+ if (nullptr != ms)
{
- fprintf(debug, "This is simulation %d, local number of ranks %d, local rank ID %d\n",
- cr->ms->sim, cr->nnodes, cr->sim_nodeid);
- fprintf(debug, "Changing to working directory %s\n", multidirs[cr->ms->sim]);
+ done_mpi_in_place_buf(ms->mpb);
+ delete ms;
}
-
- // TODO This should throw upon error
- gmx_chdir(multidirs[cr->ms->sim]);
}
#include <stdio.h>
#include "gromacs/utility/basedefinitions.h"
+#include "gromacs/utility/gmxmpi.h"
struct gmx_multisim_t;
struct t_commrec;
* no output is written.
*/
-void init_multisystem(t_commrec *cr, int nsim, char **multidirs);
+gmx_multisim_t *init_multisystem(MPI_Comm comm, int nsim, char **multidirs);
/* Splits the communication into nsim separate simulations
* and creates a communication structure between the master
* these simulations.
*/
+//! Cleans up multi-system handler.
+void done_multisim(gmx_multisim_t *ms);
+
#endif
//! Initialize the energy minimization
static void init_em(FILE *fplog, const char *title,
- t_commrec *cr, gmx::IMDOutputProvider *outputProvider,
+ t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx::IMDOutputProvider *outputProvider,
t_inputrec *ir,
const MdrunOptions &mdrunOptions,
t_state *state_global, gmx_mtop_t *top_global,
init_nrnb(nrnb);
/* Interactive molecular dynamics */
- init_IMD(ir, cr, top_global, fplog, 1,
+ init_IMD(ir, cr, ms, top_global, fplog, 1,
MASTER(cr) ? as_rvec_array(state_global->x.data()) : nullptr,
nfile, fnm, nullptr, mdrunOptions);
/* Constrain the starting coordinates */
dvdl_constr = 0;
constrain(PAR(cr) ? nullptr : fplog, TRUE, TRUE, constr, &(*top)->idef,
- ir, cr, -1, 0, 1.0, mdatoms,
+ ir, cr, ms, -1, 0, 1.0, mdatoms,
as_rvec_array(ems->s.x.data()),
as_rvec_array(ems->s.x.data()),
nullptr,
//! \brief Do one minimization step
//
// \returns true when the step succeeded, false when a constraint error occurred
-static bool do_em_step(t_commrec *cr, t_inputrec *ir, t_mdatoms *md,
+static bool do_em_step(t_commrec *cr,
+ const gmx_multisim_t *ms,
+ t_inputrec *ir, t_mdatoms *md,
gmx_bool bMolPBC,
em_state_t *ems1, real a, const PaddedRVecVector *force,
em_state_t *ems2,
dvdl_constr = 0;
validStep =
constrain(nullptr, TRUE, TRUE, constr, &top->idef,
- ir, cr, count, 0, 1.0, md,
+ ir, cr, ms, count, 0, 1.0, md,
as_rvec_array(s1->x.data()), as_rvec_array(s2->x.data()),
nullptr, bMolPBC, s2->box,
s2->lambda[efptBONDED], &dvdl_constr,
//! De one energy evaluation
static void evaluate_energy(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx_mtop_t *top_global,
em_state_t *ems, gmx_localtop_t *top,
t_inputrec *inputrec,
/* do_force always puts the charge groups in the box and shifts again
* We do not unshift, so molecules are always whole in congrad.c
*/
- do_force(fplog, cr, inputrec,
+ do_force(fplog, cr, ms, inputrec,
count, nrnb, wcycle, top, &top_global->groups,
ems->s.box, ems->s.x, &ems->s.hist,
ems->f, force_vir, mdAtoms->mdatoms(), enerd, fcd,
dvdl_constr = 0;
rvec *f_rvec = as_rvec_array(ems->f.data());
constrain(nullptr, FALSE, FALSE, constr, &top->idef,
- inputrec, cr, count, 0, 1.0, mdAtoms->mdatoms(),
+ inputrec, cr, ms, count, 0, 1.0, mdAtoms->mdatoms(),
as_rvec_array(ems->s.x.data()), f_rvec, f_rvec,
fr->bMolPBC, ems->s.box,
ems->s.lambda[efptBONDED], &dvdl_constr,
{
/*! \brief Do conjugate gradients minimization
- \copydoc integrator_t(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t(FILE *fplog, t_commrec *cr,
+ const gmx_multi_sim_t *,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
gmx_membed_t gmx_unused *membed,
gmx_walltime_accounting_t walltime_accounting)
*/
-double do_cg(FILE *fplog, t_commrec *cr, const gmx::MDLogger gmx_unused &mdlog,
+double do_cg(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger gmx_unused &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t gmx_unused *oenv,
const MdrunOptions &mdrunOptions,
em_state_t *s_c = &s3;
/* Init em and store the local state in s_min */
- init_em(fplog, CG, cr, outputProvider, inputrec, mdrunOptions,
+ init_em(fplog, CG, cr, ms, outputProvider, inputrec, mdrunOptions,
state_global, top_global, s_min, &top,
nrnb, mu_tot, fr, &enerd, &graph, mdAtoms, &gstat,
vsite, constr, nullptr,
/* do_force always puts the charge groups in the box and shifts again
* We do not unshift, so molecules are always whole in congrad.c
*/
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, s_min, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
}
/* Take a trial step (new coords in s_c) */
- do_em_step(cr, inputrec, mdatoms, fr->bMolPBC, s_min, c, &s_min->s.cg_p, s_c,
+ do_em_step(cr, ms, inputrec, mdatoms, fr->bMolPBC, s_min, c, &s_min->s.cg_p, s_c,
constr, top, nrnb, wcycle, -1);
neval++;
/* Calculate energy for the trial step */
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, s_c, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
}
/* Take a trial step to this new point - new coords in s_b */
- do_em_step(cr, inputrec, mdatoms, fr->bMolPBC, s_min, b, &s_min->s.cg_p, s_b,
+ do_em_step(cr, ms, inputrec, mdatoms, fr->bMolPBC, s_min, b, &s_min->s.cg_p, s_b,
constr, top, nrnb, wcycle, -1);
neval++;
/* Calculate energy for the trial step */
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, s_b, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
/*! \brief Do L-BFGS conjugate gradients minimization
- \copydoc integrator_t(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t(FILE *fplog, t_commrec *cr,
+ const gmx_multi_sim_t *,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
gmx_membed_t gmx_unused *membed,
gmx_walltime_accounting_t walltime_accounting)
*/
-double do_lbfgs(FILE *fplog, t_commrec *cr, const gmx::MDLogger gmx_unused &mdlog,
+double do_lbfgs(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger gmx_unused &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t gmx_unused *oenv,
const MdrunOptions &mdrunOptions,
neval = 0;
/* Init em */
- init_em(fplog, LBFGS, cr, outputProvider, inputrec, mdrunOptions,
+ init_em(fplog, LBFGS, cr, ms, outputProvider, inputrec, mdrunOptions,
state_global, top_global, &ems, &top,
nrnb, mu_tot, fr, &enerd, &graph, mdAtoms, &gstat,
vsite, constr, nullptr,
* We do not unshift, so molecules are always whole
*/
neval++;
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, &ems, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
neval++;
// Calculate energy for the trial step in position C
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, sc, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
neval++;
// Calculate energy for the trial step in point B
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, sb, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
} /* That's all folks */
/*! \brief Do steepest descents minimization
- \copydoc integrator_t(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t(FILE *fplog, t_commrec *cr,
+ const gmx_multi_sim_t *,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
const ReplicaExchangeParameters &replExParams,
gmx_walltime_accounting_t walltime_accounting)
*/
-double do_steep(FILE *fplog, t_commrec *cr, const gmx::MDLogger gmx_unused &mdlog,
+double do_steep(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger gmx_unused &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t gmx_unused *oenv,
const MdrunOptions &mdrunOptions,
em_state_t *s_try = &s1;
/* Init em and store the local state in s_try */
- init_em(fplog, SD, cr, outputProvider, inputrec, mdrunOptions,
+ init_em(fplog, SD, cr, ms, outputProvider, inputrec, mdrunOptions,
state_global, top_global, s_try, &top,
nrnb, mu_tot, fr, &enerd, &graph, mdAtoms, &gstat,
vsite, constr, nullptr,
if (count > 0)
{
validStep =
- do_em_step(cr, inputrec, mdatoms, fr->bMolPBC,
+ do_em_step(cr, ms, inputrec, mdatoms, fr->bMolPBC,
s_min, stepsize, &s_min->f, s_try,
constr, top, nrnb, wcycle, count);
}
if (validStep)
{
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, s_try, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
} /* That's all folks */
/*! \brief Do normal modes analysis
- \copydoc integrator_t(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t(FILE *fplog, t_commrec *cr,
+ const gmx_multi_sim_t *,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
const ReplicaExchangeParameters &replExParams,
gmx_walltime_accounting_t walltime_accounting)
*/
-double do_nm(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+double do_nm(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t gmx_unused *oenv,
const MdrunOptions &mdrunOptions,
em_state_t state_work {};
/* Init em and store the local state in state_minimum */
- init_em(fplog, NM, cr, outputProvider, inputrec, mdrunOptions,
+ init_em(fplog, NM, cr, ms, outputProvider, inputrec, mdrunOptions,
state_global, top_global, &state_work, &top,
nrnb, mu_tot, fr, &enerd, &graph, mdAtoms, &gstat,
vsite, constr, &shellfc,
/* Make evaluate_energy do a single node force calculation */
cr->nnodes = 1;
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, &state_work, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
if (shellfc)
{
/* Now is the time to relax the shells */
- (void) relax_shell_flexcon(fplog, cr, mdrunOptions.verbose, step,
+ (void) relax_shell_flexcon(fplog, cr, ms, mdrunOptions.verbose, step,
inputrec, bNS, force_flags,
top,
constr, enerd, fcd,
}
else
{
- evaluate_energy(fplog, cr,
+ evaluate_energy(fplog, cr, ms,
top_global, &state_work, top,
inputrec, nrnb, wcycle, gstat,
vsite, constr, fcd, graph, mdAtoms, fr,
static void init_adir(FILE *log, gmx_shellfc_t *shfc,
gmx_constr_t constr, t_idef *idef, t_inputrec *ir,
- t_commrec *cr, int dd_ac1,
+ t_commrec *cr,
+ const gmx_multisim_t *ms,
+ int dd_ac1,
gmx_int64_t step, t_mdatoms *md, int end,
rvec *x_old, rvec *x_init, rvec *x,
rvec *f, rvec *acc_dir,
}
}
}
- constrain(log, FALSE, FALSE, constr, idef, ir, cr, step, 0, 1.0, md,
+ constrain(log, FALSE, FALSE, constr, idef, ir, cr, ms, step, 0, 1.0, md,
x, xnold, nullptr, bMolPBC, box,
lambda[efptBONDED], &(dvdlambda[efptBONDED]),
nullptr, nullptr, nrnb, econqCoord);
- constrain(log, FALSE, FALSE, constr, idef, ir, cr, step, 0, 1.0, md,
+ constrain(log, FALSE, FALSE, constr, idef, ir, cr, ms, step, 0, 1.0, md,
x, xnew, nullptr, bMolPBC, box,
lambda[efptBONDED], &(dvdlambda[efptBONDED]),
nullptr, nullptr, nrnb, econqCoord);
}
/* Project the acceleration on the old bond directions */
- constrain(log, FALSE, FALSE, constr, idef, ir, cr, step, 0, 1.0, md,
+ constrain(log, FALSE, FALSE, constr, idef, ir, cr, ms, step, 0, 1.0, md,
x_old, xnew, acc_dir, bMolPBC, box,
lambda[efptBONDED], &(dvdlambda[efptBONDED]),
nullptr, nullptr, nrnb, econqDeriv_FlexCon);
}
-void relax_shell_flexcon(FILE *fplog, t_commrec *cr, gmx_bool bVerbose,
+void relax_shell_flexcon(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx_bool bVerbose,
gmx_int64_t mdstep, t_inputrec *inputrec,
gmx_bool bDoNS, int force_flags,
gmx_localtop_t *top,
{
pr_rvecs(debug, 0, "x b4 do_force", as_rvec_array(state->x.data()), homenr);
}
- do_force(fplog, cr, inputrec, mdstep, nrnb, wcycle, top, groups,
+ do_force(fplog, cr, ms, inputrec, mdstep, nrnb, wcycle, top, groups,
state->box, state->x, &state->hist,
force[Min], force_vir, md, enerd, fcd,
state->lambda, graph,
if (nflexcon)
{
init_adir(fplog, shfc,
- constr, idef, inputrec, cr, dd_ac1, mdstep, md, end,
+ constr, idef, inputrec, cr, ms, dd_ac1, mdstep, md, end,
shfc->x_old, as_rvec_array(state->x.data()), as_rvec_array(state->x.data()), as_rvec_array(force[Min].data()),
shfc->acc_dir,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
if (nflexcon)
{
init_adir(fplog, shfc,
- constr, idef, inputrec, cr, dd_ac1, mdstep, md, end,
+ constr, idef, inputrec, cr, ms, dd_ac1, mdstep, md, end,
x_old, as_rvec_array(state->x.data()), as_rvec_array(pos[Min].data()), as_rvec_array(force[Min].data()), acc_dir,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
pr_rvecs(debug, 0, "RELAX: pos[Try] ", as_rvec_array(pos[Try].data()), homenr);
}
/* Try the new positions */
- do_force(fplog, cr, inputrec, 1, nrnb, wcycle,
+ do_force(fplog, cr, ms, inputrec, 1, nrnb, wcycle,
top, groups, state->box, pos[Try], &state->hist,
force[Try], force_vir,
md, enerd, fcd, state->lambda, graph,
if (nflexcon)
{
init_adir(fplog, shfc,
- constr, idef, inputrec, cr, dd_ac1, mdstep, md, end,
+ constr, idef, inputrec, cr, ms, dd_ac1, mdstep, md, end,
x_old, as_rvec_array(state->x.data()), as_rvec_array(pos[Try].data()), as_rvec_array(force[Try].data()), acc_dir,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
struct gmx_constr;
struct gmx_enerdata_t;
struct gmx_groups_t;
+struct gmx_multisim_t;
struct gmx_shellfc_t;
struct gmx_mtop_t;
struct t_forcerec;
gmx_shellfc_t *shfc);
/* Optimize shell positions */
-void relax_shell_flexcon(FILE *log, t_commrec *cr, gmx_bool bVerbose,
+void relax_shell_flexcon(FILE *log, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx_bool bVerbose,
gmx_int64_t mdstep, t_inputrec *inputrec,
gmx_bool bDoNS, int force_flags,
gmx_localtop_t *top,
}
static void do_force_cutsVERLET(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
t_inputrec *inputrec,
gmx_int64_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_localtop_t *top,
/* Compute the bonded and non-bonded energies and optionally forces */
do_force_lowlevel(fr, inputrec, &(top->idef),
- cr, nrnb, wcycle, mdatoms,
+ cr, ms, nrnb, wcycle, mdatoms,
as_rvec_array(x.data()), hist, f, &forceWithVirial, enerd, fcd,
box, inputrec->fepvals, lambda, graph, &(top->excls), fr->mu_tot,
flags, &cycles_pme);
}
static void do_force_cutsGROUP(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
t_inputrec *inputrec,
gmx_int64_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_localtop_t *top,
/* Compute the bonded and non-bonded energies and optionally forces */
do_force_lowlevel(fr, inputrec, &(top->idef),
- cr, nrnb, wcycle, mdatoms,
+ cr, ms, nrnb, wcycle, mdatoms,
as_rvec_array(x.data()), hist, f, &forceWithVirial, enerd, fcd,
box, inputrec->fepvals, lambda,
graph, &(top->excls), fr->mu_tot,
}
void do_force(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
t_inputrec *inputrec,
gmx_int64_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_localtop_t *top,
switch (inputrec->cutoff_scheme)
{
case ecutsVERLET:
- do_force_cutsVERLET(fplog, cr, inputrec,
+ do_force_cutsVERLET(fplog, cr, ms, inputrec,
step, nrnb, wcycle,
top,
groups,
ddCloseBalanceRegion);
break;
case ecutsGROUP:
- do_force_cutsGROUP(fplog, cr, inputrec,
+ do_force_cutsGROUP(fplog, cr, ms, inputrec,
step, nrnb, wcycle,
top,
groups,
void do_constrain_first(FILE *fplog, gmx_constr_t constr,
t_inputrec *ir, t_mdatoms *md,
- t_state *state, t_commrec *cr, t_nrnb *nrnb,
+ t_state *state, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ t_nrnb *nrnb,
t_forcerec *fr, gmx_localtop_t *top)
{
int i, m, start, end;
/* constrain the current position */
constrain(nullptr, TRUE, FALSE, constr, &(top->idef),
- ir, cr, step, 0, 1.0, md,
+ ir, cr, ms, step, 0, 1.0, md,
as_rvec_array(state->x.data()), as_rvec_array(state->x.data()), nullptr,
fr->bMolPBC, state->box,
state->lambda[efptBONDED], &dvdl_dum,
/* constrain the inital velocity, and save it */
/* also may be useful if we need the ekin from the halfstep for velocity verlet */
constrain(nullptr, TRUE, FALSE, constr, &(top->idef),
- ir, cr, step, 0, 1.0, md,
+ ir, cr, ms, step, 0, 1.0, md,
as_rvec_array(state->x.data()), as_rvec_array(state->v.data()), as_rvec_array(state->v.data()),
fr->bMolPBC, state->box,
state->lambda[efptBONDED], &dvdl_dum,
}
dvdl_dum = 0;
constrain(nullptr, TRUE, FALSE, constr, &(top->idef),
- ir, cr, step, -1, 1.0, md,
+ ir, cr, ms, step, -1, 1.0, md,
as_rvec_array(state->x.data()), savex, nullptr,
fr->bMolPBC, state->box,
state->lambda[efptBONDED], &dvdl_dum,
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
struct gmx_constr;
struct gmx_localtop_t;
+struct gmx_multisim_t;
struct gmx_output_env_t;
struct gmx_update_t;
struct MdrunOptions;
void do_constrain_first(FILE *log, gmx_constr *constr,
t_inputrec *inputrec, t_mdatoms *md,
- t_state *state, t_commrec *cr, t_nrnb *nrnb,
+ t_state *state, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ t_nrnb *nrnb,
t_forcerec *fr, gmx_localtop_t *top);
void init_md(FILE *fplog,
namespace gmx
{
-SimulationSignaller::SimulationSignaller(SimulationSignals *signals,
- const t_commrec *cr,
- bool doInterSim,
- bool doIntraSim)
- : signals_(signals), cr_(cr),
+SimulationSignaller::SimulationSignaller(SimulationSignals *signals,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ bool doInterSim,
+ bool doIntraSim)
+ : signals_(signals), cr_(cr), ms_(ms),
doInterSim_(doInterSim),
doIntraSim_(doInterSim || doIntraSim),
mpiBuffer_ {}
// multi-simulation begin active should already have issued an
// error at mdrun time in release mode, so there's no need for a
// release-mode assertion.
- GMX_ASSERT(isMultiSim(cr_->ms), "Cannot do inter-simulation signalling without a multi-simulation");
+ GMX_ASSERT(isMultiSim(ms_), "Cannot do inter-simulation signalling without a multi-simulation");
if (MASTER(cr_))
{
// Communicate the signals between the simulations.
- gmx_sum_sim(eglsNR, mpiBuffer_.data(), cr_->ms);
+ gmx_sum_sim(eglsNR, mpiBuffer_.data(), ms_);
}
// Communicate the signals from the master to the others.
gmx_bcast(eglsNR*sizeof(mpiBuffer_[0]), mpiBuffer_.data(), cr_);
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2011,2014,2015,2016, by the GROMACS development team, led by
+ * Copyright (c) 2011,2014,2015,2016,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "gromacs/utility/real.h"
+struct gmx_multisim_t;
struct t_commrec;
//! Kinds of simulation conditions to signal about.
{
public:
//! Constructor
- SimulationSignaller(SimulationSignals *signals,
- const t_commrec *cr,
- bool doInterSim,
- bool doIntraSim);
+ SimulationSignaller(SimulationSignals *signals,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ bool doInterSim,
+ bool doIntraSim);
/*! \brief Return a reference to an array of signal values to communicate.
*
* \return If intra-sim signalling will take place, fill and
SimulationSignals *signals_;
//! Communication object.
const t_commrec *cr_;
+ //! Multi-sim handler.
+ const gmx_multisim_t *ms_;
//! Do inter-sim communication at this step.
bool doInterSim_;
//! Do intra-sim communication at this step.
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2016, by the GROMACS development team, led by
+ * Copyright (c) 2016,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
//! Check that a null signaller can be called without problems
TEST(NullSignalTest, NullSignallerWorks)
{
- SimulationSignaller signaller(nullptr, nullptr, false, false);
+ SimulationSignaller signaller(nullptr, nullptr, nullptr, false, false);
EXPECT_EQ(0, signaller.getCommunicationBuffer().size());
signaller.finalizeSignals();
}
TEST_F(SignalTest, NoSignalPropagatesIfNoSignallingTakesPlace)
{
- SimulationSignaller signaller(&signals_, nullptr, false, false);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, false, false);
EXPECT_EQ(0, signaller.getCommunicationBuffer().size());
signaller.finalizeSignals();
EXPECT_EQ(1, signals_[0].sig);
TEST_F(SignalTest, LocalIntraSimSignalPropagatesWhenIntraSimSignalTakesPlace)
{
- SimulationSignaller signaller(&signals_, nullptr, false, true);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, false, true);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
signaller.finalizeSignals();
EXPECT_EQ(0, signals_[0].sig);
TEST_F(SignalTest, LocalIntraSimSignalPropagatesWhenInterSimTakesPlace)
{
- SimulationSignaller signaller(&signals_, nullptr, true, false);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, true, false);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
// Can't call finalizeSignals without a full commrec
signaller.setSignals();
TEST_F(SignalTest, LocalIntraSimSignalPropagatesWhenBothTakePlace)
{
- SimulationSignaller signaller(&signals_, nullptr, true, true);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, true, true);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
// Can't call finalizeSignals without a full commrec
signaller.setSignals();
TEST_F(SignalTest, NonLocalSignalDoesntPropagateWhenIntraSimSignalTakesPlace)
{
signals_[0].isLocal = false;
- SimulationSignaller signaller(&signals_, nullptr, false, true);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, false, true);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
signaller.finalizeSignals();
EXPECT_EQ(1, signals_[0].sig);
TEST_F(SignalTest, NonLocalSignalPropagatesWhenInterSimSignalTakesPlace)
{
signals_[0].isLocal = false;
- SimulationSignaller signaller(&signals_, nullptr, true, false);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, true, false);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
// Can't call finalizeSignals without a full commrec
signaller.setSignals();
TEST_F(SignalTest, NonLocalSignalPropagatesWhenBothTakePlace)
{
signals_[0].isLocal = false;
- SimulationSignaller signaller(&signals_, nullptr, true, true);
+ SimulationSignaller signaller(&signals_, nullptr, nullptr, true, true);
EXPECT_NE(0, signaller.getCommunicationBuffer().size());
// Can't call finalizeSignals without a full commrec
signaller.setSignals();
{
/*! \brief Do test particle insertion.
- \copydoc integrator_t (FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t (FILE *fplog, t_commrec *cr,
+ const gmx_multi_sim_t *,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
gmx_membed_t gmx_unused *membed,
gmx_walltime_accounting_t walltime_accounting)
*/
-double do_tpi(FILE *fplog, t_commrec *cr, const gmx::MDLogger gmx_unused &mdlog,
+double do_tpi(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger gmx_unused &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
* out of the box. */
/* Make do_force do a single node force calculation */
cr->nnodes = 1;
- do_force(fplog, cr, inputrec,
+ do_force(fplog, cr, ms, inputrec,
step, nrnb, wcycle, top, &top_global->groups,
state_global->box, state_global->x, &state_global->hist,
f, force_vir, mdatoms, enerd, fcd,
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
t_idef *idef,
tensor vir_part,
t_commrec *cr,
+ const gmx_multisim_t *ms,
t_nrnb *nrnb,
gmx_wallcycle_t wcycle,
gmx_update_t *upd,
if (EI_VV(inputrec->eI) && bFirstHalf)
{
constrain(nullptr, bLog, bEner, constr, idef,
- inputrec, cr, step, 1, 1.0, md,
+ inputrec, cr, ms, step, 1, 1.0, md,
as_rvec_array(state->x.data()), as_rvec_array(state->v.data()), as_rvec_array(state->v.data()),
bMolPBC, state->box,
state->lambda[efptBONDED], dvdlambda,
else
{
constrain(nullptr, bLog, bEner, constr, idef,
- inputrec, cr, step, 1, 1.0, md,
+ inputrec, cr, ms, step, 1, 1.0, md,
as_rvec_array(state->x.data()), as_rvec_array(upd->xp.data()), nullptr,
bMolPBC, state->box,
state->lambda[efptBONDED], dvdlambda,
wallcycle_start(wcycle, ewcCONSTR);
constrain(nullptr, bLog, bEner, constr, idef,
- inputrec, cr, step, 1, 0.5, md,
+ inputrec, cr, ms, step, 1, 0.5, md,
as_rvec_array(state->x.data()), as_rvec_array(upd->xp.data()), nullptr,
bMolPBC, state->box,
state->lambda[efptBONDED], dvdlambda,
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
struct gmx_constr;
struct gmx_ekindata_t;
struct gmx_enerdata_t;
+struct gmx_multisim_t;
struct t_extmass;
struct t_fcdata;
struct t_graph;
t_idef *idef,
tensor vir_part,
t_commrec *cr,
+ const gmx_multisim_t *ms,
t_nrnb *nrnb,
gmx_wallcycle_t wcycle,
gmx_update_t *upd,
/* This routine cannot print tons of data, since it is called before the log file is opened. */
void
-handleRestart(t_commrec *cr,
- gmx_bool bTryToAppendFiles,
- const int NFILE,
- t_filenm fnm[],
- bool *bDoAppendFiles,
- bool *bStartFromCpt)
+handleRestart(t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx_bool bTryToAppendFiles,
+ const int NFILE,
+ t_filenm fnm[],
+ bool *bDoAppendFiles,
+ bool *bStartFromCpt)
{
gmx_bool bAddPart;
int sim_part, sim_part_fn;
&sim_part_fn, cr,
bTryToAppendFiles, NFILE, fnm,
part_suffix, &bAddPart, bDoAppendFiles);
- if (sim_part_fn == 0 && isMasterSimMasterRank(cr->ms, cr))
+ if (sim_part_fn == 0 && isMasterSimMasterRank(ms, cr))
{
fprintf(stdout, "No previous checkpoint file present with -cpi option, assuming this is a new run.\n");
}
// Master ranks of multi simulations should check that the
// simulation part number is consistent across the
// simulations.
- if (isMultiSim(cr->ms) && MASTER(cr))
+ if (isMultiSim(ms) && MASTER(cr))
{
// Only the master simulation should report on problems.
- if (isMasterSimMasterRank(cr->ms, cr))
+ if (isMasterSimMasterRank(ms, cr))
{
/* Log file is not yet available, so if there's a
* problem we can only write to stderr. */
{
fpmulti = nullptr;
}
- check_multi_int(fpmulti, cr->ms, sim_part, "simulation part", TRUE);
+ check_multi_int(fpmulti, ms, sim_part, "simulation part", TRUE);
}
}
else
sprintf(suffix, "%s%04d", part_suffix, sim_part_fn);
add_suffix_to_output_names(fnm, NFILE, suffix);
- if (isMasterSimMasterRank(cr->ms, cr))
+ if (isMasterSimMasterRank(ms, cr))
{
fprintf(stdout, "Checkpoint file is from part %d, new output files will be suffixed '%s'.\n", sim_part-1, suffix);
}
#include "gromacs/utility/basedefinitions.h"
+struct gmx_multisim_t;
struct t_commrec;
struct t_filenm;
* and/or simulations.
*
* \param[in] cr Communication structure
+ * \param[in] ms Handles multi-simulations.
* \param[in] bTryToAppendFiles Whether appending is requested (from mdrun)
* \param[in] NFILE Size of fnm struct
* \param[inout] fnm Filename parameters to mdrun
* \param[out] bStartFromCpt True on return if we found the checkpoint
* and will use it to restart.
*/
-void handleRestart(t_commrec *cr,
- gmx_bool bTryToAppendFiles,
- const int NFILE,
- t_filenm fnm[],
- bool *bDoAppendFiles,
- bool *bStartFromCpt);
+void handleRestart(t_commrec *cr,
+ const gmx_multisim_t *ms,
+ gmx_bool bTryToAppendFiles,
+ const int NFILE,
+ t_filenm fnm[],
+ bool *bDoAppendFiles,
+ bool *bStartFromCpt);
#endif
setLogicalProcessorCount(1);
}
int numThreadsOnThisNode, indexWithinNodeOfFirstThreadOnThisRank;
- analyzeThreadsOnThisNode(cr_, &affinityAccess_,
+ analyzeThreadsOnThisNode(cr_, nullptr, &affinityAccess_,
numThreadsOnThisRank,
&numThreadsOnThisNode,
&indexWithinNodeOfFirstThreadOnThisRank);
}
void analyzeThreadsOnThisNode(const t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx::IThreadAffinityAccess *affinityAccess,
int numThreadsOnThisRank,
int *numThreadsOnThisNode,
*intraNodeThreadOffset = 0;
*numThreadsOnThisNode = numThreadsOnThisRank;
#if GMX_MPI
- if (PAR(cr) || isMultiSim(cr->ms))
+ if (PAR(cr) || isMultiSim(ms))
{
if (affinityAccess == nullptr)
{
}
#else
GMX_UNUSED_VALUE(cr);
+ GMX_UNUSED_VALUE(ms);
GMX_UNUSED_VALUE(affinityAccess);
#endif
#include "gromacs/utility/basedefinitions.h"
struct gmx_hw_opt_t;
+struct gmx_multisim_t;
struct t_commrec;
namespace gmx
* See gmx_set_thread_affinity(), which consumes this output.
*
* \param[in] cr Communication handler.
+ * \param[in] ms Multi-simulation handler.
* \param[in] affinityAccess Interface for low-level access to affinity details.
* \param[in] numThreadsOnThisRank The number of threads on this rank.
* \param[out] numThreadsOnThisNode On exit, the number of threads on all ranks of this node.
* in the set of all the threads of all MPI ranks within a node (ordered by MPI rank ID).
*/
void analyzeThreadsOnThisNode(const t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx::IThreadAffinityAccess *affinityAccess,
int numThreadsOnThisRank,
int *numThreadsOnThisNode,
int dbuf_alloc;
} mpi_in_place_buf_t;
-struct gmx_multisim_t {
- int nsim;
- int sim;
- MPI_Group mpi_group_masters;
- MPI_Comm mpi_comm_masters;
+void done_mpi_in_place_buf(mpi_in_place_buf_t *buf);
+
+struct gmx_multisim_t
+{
+ int nsim = 1;
+ int sim = 0;
+ MPI_Group mpi_group_masters = MPI_GROUP_NULL;
+ MPI_Comm mpi_comm_masters = MPI_COMM_NULL;
/* these buffers are used as destination buffers if MPI_IN_PLACE isn't
supported.*/
- mpi_in_place_buf_t *mpb;
+ mpi_in_place_buf_t *mpb = nullptr;
};
#define DUTY_PP (1<<0)
*/
int duty;
- gmx_multisim_t *ms;
-
/* these buffers are used as destination buffers if MPI_IN_PLACE isn't
supported.*/
mpi_in_place_buf_t *mpb;
}
}
-void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t *hw_opt,
- const gmx_hw_info_t &hwinfo,
- const t_commrec *cr,
- PmeRunMode pmeRunMode,
- const gmx_mtop_t &mtop)
+void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t *hw_opt,
+ const gmx_hw_info_t &hwinfo,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ PmeRunMode pmeRunMode,
+ const gmx_mtop_t &mtop)
{
#if GMX_THREAD_MPI
GMX_RELEASE_ASSERT(hw_opt->nthreads_tmpi >= 1, "Must have at least one thread-MPI rank");
* all detected ncore_tot physical cores. We are currently not
* checking for that here.
*/
- int numRanksTot = cr->nnodes*(isMultiSim(cr->ms) ? cr->ms->nsim : 1);
+ int numRanksTot = cr->nnodes*(isMultiSim(ms) ? ms->nsim : 1);
int numAtomsPerRank = mtop.natoms/cr->nnodes;
int numCoresPerRank = hwinfo.ncore_tot/numRanksTot;
if (numAtomsPerRank < c_numAtomsPerCoreSquaredSmtThreshold*gmx::square(numCoresPerRank))
void checkHardwareOversubscription(int numThreadsOnThisRank,
const gmx::HardwareTopology &hwTop,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const gmx::MDLogger &mdlog)
{
if (hwTop.supportLevel() < gmx::HardwareTopology::SupportLevel::LogicalProcessorCount)
int numRanksOnThisNode = 1;
int numThreadsOnThisNode = numThreadsOnThisRank;
#if GMX_MPI
- if (PAR(cr) || isMultiSim(cr->ms))
+ if (PAR(cr) || isMultiSim(ms))
{
/* Count the threads within this physical node */
MPI_Comm_size(cr->mpi_comm_physicalnode, &numRanksOnThisNode);
MPI_Allreduce(&numThreadsOnThisRank, &numThreadsOnThisNode, 1, MPI_INT, MPI_SUM, cr->mpi_comm_physicalnode);
}
+#else
+ GMX_UNUSED_VALUE(ms);
#endif
if (numThreadsOnThisNode > hwTop.machine().logicalProcessorCount)
struct gmx_hw_info_t;
struct gmx_hw_opt_t;
struct gmx_mtop_t;
+struct gmx_multisim_t;
struct t_commrec;
struct t_inputrec;
*
* Should be called when we know the MPI rank count and PME run mode.
*/
-void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t *hw_opt,
- const gmx_hw_info_t &hwinfo,
- const t_commrec *cr,
- PmeRunMode pmeRunMode,
- const gmx_mtop_t &mtop);
+void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t *hw_opt,
+ const gmx_hw_info_t &hwinfo,
+ const t_commrec *cr,
+ const gmx_multisim_t *ms,
+ PmeRunMode pmeRunMode,
+ const gmx_mtop_t &mtop);
/*! \brief Warns for oversubscribing the hardware threads, when that is the case
*/
void checkHardwareOversubscription(int numThreadsOnThisRank,
const gmx::HardwareTopology &hwTop,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const gmx::MDLogger &mdlog);
#endif
const gmx_hw_info_t &hardwareInfo,
const MDLogger &mdlog,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const std::vector<GpuTask> &gpuTasksOnThisRank)
{
/* Communicate among ranks on this node to find each task that can
MPI_Barrier(cr->mpi_comm_mysim);
#endif
}
- if (isMultiSim(cr->ms))
+ if (isMultiSim(ms))
{
#if GMX_MPI
- MPI_Barrier(cr->ms->mpi_comm_masters);
+ MPI_Barrier(ms->mpi_comm_masters);
#endif
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2017, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include <vector>
struct gmx_hw_info_t;
+struct gmx_multisim_t;
struct t_commrec;
namespace gmx
* \param[in] hardwareInfo The detected hardware
* \param[in] mdlog Logging object to write to.
* \param[in] cr Communication object.
+ * \param[in] ms Handles multi-simulations.
* \param[in] gpuTasksOnThisRank Information about what GPU tasks
* exist on this rank.
*
const gmx_hw_info_t &hardwareInfo,
const MDLogger &mdlog,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
const std::vector<GpuTask> &gpuTasksOnThisRank);
//! Function for whether the task of \c mapping has value \c TaskType.
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2013,2014,2016,2017, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2016,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
typedef void* MPI_Status;
typedef void* MPI_Group;
#define MPI_COMM_NULL NULL
+#define MPI_GROUP_NULL NULL
+#define MPI_COMM_WORLD NULL
#endif
#endif
//! \endcond
}
/*! \libinternal
- \copydoc integrator_t (FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+ \copydoc integrator_t (FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
gmx_membed_t *membed,
gmx_walltime_accounting_t walltime_accounting)
*/
-double gmx::do_md(FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
+double gmx::do_md(FILE *fplog, t_commrec *cr,
+ const gmx_multisim_t *ms,
+ const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv,
const MdrunOptions &mdrunOptions,
SimulationSignals signals;
// Most global communnication stages don't propagate mdrun
// signals, and will use this object to achieve that.
- SimulationSignaller nullSignaller(nullptr, nullptr, false, false);
+ SimulationSignaller nullSignaller(nullptr, nullptr, nullptr, false, false);
if (!mdrunOptions.writeConfout)
{
}
/* Set up interactive MD (IMD) */
- init_IMD(ir, cr, top_global, fplog, ir->nstcalcenergy, MASTER(cr) ? as_rvec_array(state_global->x.data()) : nullptr,
+ init_IMD(ir, cr, ms, top_global, fplog, ir->nstcalcenergy,
+ MASTER(cr) ? as_rvec_array(state_global->x.data()) : nullptr,
nfile, fnm, oenv, mdrunOptions);
if (DOMAINDECOMP(cr))
/* Initialize AWH and restore state from history in checkpoint if needed. */
if (ir->bDoAwh)
{
- ir->awh = new gmx::Awh(fplog, *ir, cr, *ir->awhParams, opt2fn("-awh", nfile, fnm), ir->pull_work);
+ ir->awh = new gmx::Awh(fplog, *ir, cr, ms, *ir->awhParams, opt2fn("-awh", nfile, fnm), ir->pull_work);
if (startingFromCheckpoint)
{
const bool useReplicaExchange = (replExParams.exchangeInterval > 0);
if (useReplicaExchange && MASTER(cr))
{
- repl_ex = init_replica_exchange(fplog, cr->ms, top_global->natoms, ir,
+ repl_ex = init_replica_exchange(fplog, ms, top_global->natoms, ir,
replExParams);
}
{
/* Constrain the initial coordinates and velocities */
do_constrain_first(fplog, constr, ir, mdatoms, state,
- cr, nrnb, fr, top);
+ cr, ms, nrnb, fr, top);
}
if (vsite)
{
bNeedRepartition = FALSE;
// TODO This implementation of ensemble orientation restraints is nasty because
// a user can't just do multi-sim with single-sim orientation restraints.
- bUsingEnsembleRestraints = (fcd->disres.nsystems > 1) || (cr->ms && fcd->orires.nr);
+ bUsingEnsembleRestraints = (fcd->disres.nsystems > 1) || (ms && fcd->orires.nr);
{
// Replica exchange and ensemble restraints need all
step_rel = 0;
// TODO extract this to new multi-simulation module
- if (MASTER(cr) && isMultiSim(cr->ms) && !useReplicaExchange)
+ if (MASTER(cr) && isMultiSim(ms) && !useReplicaExchange)
{
- if (!multisim_int_all_are_equal(cr->ms, ir->nsteps))
+ if (!multisim_int_all_are_equal(ms, ir->nsteps))
{
GMX_LOG(mdlog.warning).appendText(
"Note: The number of steps is not consistent across multi simulations,\n"
"but we are proceeding anyway!");
}
- if (!multisim_int_all_are_equal(cr->ms, ir->init_step))
+ if (!multisim_int_all_are_equal(ms, ir->init_step))
{
GMX_LOG(mdlog.warning).appendText(
"Note: The initial step is not consistent across multi simulations,\n"
if (shellfc)
{
/* Now is the time to relax the shells */
- relax_shell_flexcon(fplog, cr, mdrunOptions.verbose, step,
+ relax_shell_flexcon(fplog, cr, ms, mdrunOptions.verbose, step,
ir, bNS, force_flags, top,
constr, enerd, fcd,
state, &f, force_vir, mdatoms,
* This is parallellized as well, and does communication too.
* Check comments in sim_util.c
*/
- do_force(fplog, cr, ir, step, nrnb, wcycle, top, groups,
+ do_force(fplog, cr, ms, ir, step, nrnb, wcycle, top, groups,
state->box, state->x, &state->hist,
f, force_vir, mdatoms, enerd, fcd,
state->lambda, graph,
update_constraints(fplog, step, nullptr, ir, mdatoms,
state, fr->bMolPBC, graph, f,
&top->idef, shake_vir,
- cr, nrnb, wcycle, upd, constr,
+ cr, ms, nrnb, wcycle, upd, constr,
TRUE, bCalcVir);
wallcycle_start(wcycle, ewcUPDATE);
}
update_constraints(fplog, step, nullptr, ir, mdatoms,
state, fr->bMolPBC, graph, f,
&top->idef, tmp_vir,
- cr, nrnb, wcycle, upd, constr,
+ cr, ms, nrnb, wcycle, upd, constr,
TRUE, bCalcVir);
}
}
update_constraints(fplog, step, &dvdl_constr, ir, mdatoms, state,
fr->bMolPBC, graph, f,
&top->idef, shake_vir,
- cr, nrnb, wcycle, upd, constr,
+ cr, ms, nrnb, wcycle, upd, constr,
FALSE, bCalcVir);
if (ir->eI == eiVVAK)
update_constraints(fplog, step, nullptr, ir, mdatoms,
state, fr->bMolPBC, graph, f,
&top->idef, tmp_vir,
- cr, nrnb, wcycle, upd, nullptr,
+ cr, ms, nrnb, wcycle, upd, nullptr,
FALSE, bCalcVir);
}
if (EI_VV(ir->eI))
// situation where e.g. checkpointing can't be
// signalled.
bool doIntraSimSignal = true;
- SimulationSignaller signaller(&signals, cr, doInterSimSignal, doIntraSimSignal);
+ SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal);
compute_globals(fplog, gstat, cr, ir, fr, ekind, state, mdatoms, nrnb, vcm,
wcycle, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
state->fep_state = lamnew;
}
/* Print the remaining wall clock time for the run */
- if (isMasterSimMasterRank(cr->ms, cr) &&
+ if (isMasterSimMasterRank(ms, cr) &&
(do_verbose || gmx_got_usr_signal()) &&
!bPMETunePrinting)
{
bExchanged = FALSE;
if (bDoReplEx)
{
- bExchanged = replica_exchange(fplog, cr, repl_ex,
+ bExchanged = replica_exchange(fplog, cr, ms, repl_ex,
state_global, enerd,
state, step, t);
}
*/
#include "gmxpre.h"
+#include "config.h"
+
#include <cstdio>
#include <cstdlib>
#include <cstring>
hw_opt.thread_affinity = nenum(thread_aff_opt_choices);
// now check for a multi-simulation
- int nmultisim = 0;
+ int nmultisim = 1;
if (opt2bSet("-multidir", nfile, fnm))
{
nmultisim = opt2fns(&multidir, "-multidir", nfile, fnm);
gmx_fatal(FARGS, "Replica exchange number of exchanges needs to be positive");
}
- init_multisystem(cr, nmultisim, multidir);
+ ms = init_multisystem(MPI_COMM_WORLD, nmultisim, multidir);
+
+ /* Prepare the intra-simulation communication */
+ // TODO consolidate this with init_commrec, after changing the
+ // relative ordering of init_commrec and init_multisystem
+#if GMX_MPI
+ if (ms != nullptr)
+ {
+ cr->nnodes = cr->nnodes / nmultisim;
+ MPI_Comm_split(MPI_COMM_WORLD, ms->sim, cr->sim_nodeid, &cr->mpi_comm_mysim);
+ cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
+ MPI_Comm_rank(cr->mpi_comm_mysim, &cr->sim_nodeid);
+ MPI_Comm_rank(cr->mpi_comm_mygroup, &cr->nodeid);
+ }
+#endif
if (!opt2bSet("-cpi", nfile, fnm))
{
continuationOptions.appendFilesOptionSet = opt2parg_bSet("-append", asize(pa), pa);
- handleRestart(cr, bTryToAppendFiles, nfile, fnm, &continuationOptions.appendFiles, &continuationOptions.startedFromCheckpoint);
+ handleRestart(cr, ms, bTryToAppendFiles, nfile, fnm, &continuationOptions.appendFiles, &continuationOptions.startedFromCheckpoint);
mdrunOptions.rerun = opt2bSet("-rerun", nfile, fnm);
mdrunOptions.ntompOptionIsSet = opt2parg_bSet("-ntomp", asize(pa), pa);
/* Log file has to be closed in mdrunner if we are appending to it
(fplog not set here) */
- if (MASTER(cr) && !continuationOptions.appendFiles)
+ if (fplog != nullptr)
{
gmx_log_close(fplog);
}
+ if (GMX_LIB_MPI)
+ {
+ done_commrec(cr);
+ }
+ done_multisim(ms);
return rc;
}
{
gmx_fatal(FARGS, "Replica exchange is only supported by dynamical simulations");
/* Note that PAR(cr) is defined by cr->nnodes > 1, which is
- * distinct from isMultiSim(cr->ms). A multi-simulation only runs
+ * distinct from isMultiSim(ms). A multi-simulation only runs
* with real MPI parallelism, but this does not imply PAR(cr)
* is true!
*
}
}
-gmx_bool replica_exchange(FILE *fplog, const t_commrec *cr, struct gmx_repl_ex *re,
+gmx_bool replica_exchange(FILE *fplog, const t_commrec *cr,
+ const gmx_multisim_t *ms, struct gmx_repl_ex *re,
t_state *state, const gmx_enerdata_t *enerd,
t_state *state_local, gmx_int64_t step, real time)
{
if (MASTER(cr))
{
replica_id = re->repl;
- test_for_replica_exchange(fplog, cr->ms, re, enerd, det(state_local->box), step, time);
+ test_for_replica_exchange(fplog, ms, re, enerd, det(state_local->box), step, time);
prepare_to_do_exchange(re, replica_id, &maxswap, &bThisReplicaExchanged);
}
/* Do intra-simulation broadcast so all processors belonging to
{
fprintf(debug, "Exchanging %d with %d\n", replica_id, exchange_partner);
}
- exchange_state(cr->ms, exchange_partner, state);
+ exchange_state(ms, exchange_partner, state);
}
}
/* For temperature-type replica exchange, we need to scale
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2011,2012,2013,2014,2015,2017, by the GROMACS development team, led by
+ * Copyright (c) 2011,2012,2013,2014,2015,2017,2018, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
gmx_bool replica_exchange(FILE *fplog,
const t_commrec *cr,
+ const gmx_multisim_t *ms,
gmx_repl_ex_t re,
t_state *state, const gmx_enerdata_t *enerd,
t_state *state_local,
// Mdrunner.
fnm = dup_tfn(nfile, fnm);
- cr = reinitialize_commrec_for_this_thread(cr);
+ cr = reinitialize_commrec_for_this_thread(cr, ms);
if (!MASTER(cr))
{
GMX_UNUSED_VALUE(mdrunner_start_fn);
#endif
- return reinitialize_commrec_for_this_thread(cr);
+ return reinitialize_commrec_for_this_thread(cr, ms);
}
} // namespace
hwinfo = gmx_detect_hardware(mdlog);
- gmx_print_detected_hardware(fplog, cr, mdlog, hwinfo);
+ gmx_print_detected_hardware(fplog, cr, ms, mdlog, hwinfo);
std::vector<int> gpuIdsToUse;
auto compatibleGpus = getCompatibleGpus(hwinfo->gpu_info);
snew(fcd, 1);
/* This needs to be called before read_checkpoint to extend the state */
- init_disres(fplog, mtop, inputrec, cr, fcd, globalState.get(), replExParams.exchangeInterval > 0);
+ init_disres(fplog, mtop, inputrec, cr, ms, fcd, globalState.get(), replExParams.exchangeInterval > 0);
- init_orires(fplog, mtop, inputrec, cr, globalState.get(), &(fcd->orires));
+ init_orires(fplog, mtop, inputrec, cr, ms, globalState.get(), &(fcd->orires));
if (inputrecDeform(inputrec))
{
/* Initialize per-physical-node MPI process/thread ID and counters. */
gmx_init_intranode_counters(cr);
#if GMX_MPI
- if (isMultiSim(cr->ms))
+ if (isMultiSim(ms))
{
GMX_LOG(mdlog.warning).asParagraph().appendTextFormatted(
"This is simulation %d out of %d running as a composite GROMACS\n"
"multi-simulation job. Setup for this simulation:\n",
- cr->ms->sim, cr->ms->nsim);
+ ms->sim, ms->nsim);
}
GMX_LOG(mdlog.warning).appendTextFormatted(
"Using %d MPI %s\n",
check_and_update_hw_opt_2(&hw_opt, inputrec->cutoff_scheme);
/* Check and update the number of OpenMP threads requested */
- checkAndUpdateRequestedNumOpenmpThreads(&hw_opt, *hwinfo, cr, pmeRunMode, *mtop);
+ checkAndUpdateRequestedNumOpenmpThreads(&hw_opt, *hwinfo, cr, ms, pmeRunMode, *mtop);
gmx_omp_nthreads_init(mdlog, cr,
hwinfo->nthreads_hw_avail,
{
// Produce the task assignment for this rank.
gpuTaskAssignment = runTaskAssignment(gpuIdsToUse, userGpuTaskAssignment, *hwinfo,
- mdlog, cr, gpuTasksOnThisRank);
+ mdlog, cr, ms, gpuTasksOnThisRank);
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
{
MPI_Barrier(cr->mpi_comm_mysim);
}
- if (isMultiSim(cr->ms))
+ if (isMultiSim(ms))
{
- MPI_Barrier(cr->ms->mpi_comm_masters);
+ MPI_Barrier(ms->mpi_comm_masters);
}
#endif
checkHardwareOversubscription(numThreadsOnThisRank,
*hwinfo->hardwareTopology,
- cr, mdlog);
+ cr, ms, mdlog);
if (hw_opt.thread_affinity != threadaffOFF)
{
&hw_opt, hwinfo->nthreads_hw_avail, TRUE);
int numThreadsOnThisNode, intraNodeThreadOffset;
- analyzeThreadsOnThisNode(cr, nullptr, numThreadsOnThisRank, &numThreadsOnThisNode,
+ analyzeThreadsOnThisNode(cr, ms, nullptr, numThreadsOnThisRank, &numThreadsOnThisNode,
&intraNodeThreadOffset);
/* Set the CPU affinity */
}
/* Now do whatever the user wants us to do (how flexible...) */
- my_integrator(inputrec->eI) (fplog, cr, mdlog, nfile, fnm,
+ my_integrator(inputrec->eI) (fplog, cr, ms, mdlog, nfile, fnm,
oenv,
mdrunOptions,
vsite, constr,
inputrec, nrnb, wcycle, walltime_accounting,
fr ? fr->nbv : nullptr,
pmedata,
- EI_DYNAMICS(inputrec->eI) && !isMultiSim(cr->ms));
+ EI_DYNAMICS(inputrec->eI) && !isMultiSim(ms));
// Free PME data
if (pmedata)
mdModules.reset(nullptr); // destruct force providers here as they might also use the GPU
/* Free GPU memory and set a physical node tMPI barrier (which should eventually go away) */
- free_gpu_resources(fr, cr);
+ free_gpu_resources(fr, cr, ms);
free_gpu(nonbondedDeviceInfo);
free_gpu(pmeDeviceInfo);
if (MASTER(cr) && continuationOptions.appendFiles)
{
gmx_log_close(fplog);
+ fplog = nullptr;
}
rc = (int)gmx_get_stop_condition();
wait for that. */
if (PAR(cr) && MASTER(cr))
{
+ done_commrec(cr);
tMPI_Finalize();
}
#endif
FILE *fplog;
//! Handle to communication data structure.
t_commrec *cr;
+ //! Handle to multi-simulation handler.
+ gmx_multisim_t *ms;
public:
/*! \brief Defaulted constructor.
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff