From 3120011062ce1c7e1b807e1abd6026f499cf3d7f Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Wed, 4 Sep 2019 08:45:19 +0200 Subject: [PATCH] Move constraint and bonded filtering info into DDSystemInfo Change-Id: Ic3cc4b5da309ab3f8e719739a5a162cd2c2d62bf --- src/gromacs/domdec/domdec.cpp | 44 +++++++++++++---------- src/gromacs/domdec/domdec.h | 3 ++ src/gromacs/domdec/domdec_constraints.cpp | 7 ++-- src/gromacs/domdec/domdec_internal.h | 12 +++++-- src/gromacs/domdec/domdec_struct.h | 5 --- src/gromacs/domdec/domdec_topology.cpp | 5 +-- src/gromacs/domdec/partition.cpp | 4 +-- src/gromacs/mdlib/constr.cpp | 4 +-- 8 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index 71dbfb60b5..37514e9ef2 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -667,7 +667,7 @@ real dd_cutoff_multibody(const gmx_domdec_t *dd) { r = std::min(r, comm.cellsize_min[dd->dim[di]]); } - if (comm.bBondComm) + if (comm.systemInfo.filterBondedCommunication) { r = std::max(r, comm.cutoff_mbody); } @@ -1081,6 +1081,11 @@ int dd_pme_maxshift_y(const gmx_domdec_t *dd) } } +bool ddHaveSplitConstraints(const gmx_domdec_t &dd) +{ + return dd.comm->systemInfo.haveSplitConstraints; +} + void dd_cycles_add(const gmx_domdec_t *dd, float cycles, int ddCycl) { /* Note that the cycles value can be incorrect, either 0 or some @@ -2167,13 +2172,13 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, if (systemInfo.useUpdateGroups) { - dd->splitConstraints = false; - dd->splitSettles = false; + systemInfo.haveSplitConstraints = false; + systemInfo.haveSplitSettles = false; } else { - dd->splitConstraints = gmx::inter_charge_group_constraints(*mtop); - dd->splitSettles = gmx::inter_charge_group_settles(*mtop); + systemInfo.haveSplitConstraints = gmx::inter_charge_group_constraints(*mtop); + systemInfo.haveSplitSettles = gmx::inter_charge_group_settles(*mtop); } if (ir->rlist == 0) @@ -2191,8 +2196,8 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, systemInfo.minCutoffForMultiBody = 0; /* Determine the minimum cell size limit, affected by many factors */ - systemInfo.cellsizeLimit = 0; - comm->bBondComm = FALSE; + systemInfo.cellsizeLimit = 0; + systemInfo.filterBondedCommunication = false; /* We do not allow home atoms to move beyond the neighboring domain * between domain decomposition steps, which limits the cell size. @@ -2230,7 +2235,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, atomToAtomIntoDomainToDomainCutoff(systemInfo, options.minimumCommunicationRange); if (options.useBondedCommunication) { - comm->bBondComm = (systemInfo.minCutoffForMultiBody > systemInfo.cutoff); + systemInfo.filterBondedCommunication = (systemInfo.minCutoffForMultiBody > systemInfo.cutoff); } else { @@ -2268,7 +2273,8 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, { r_bonded = std::max(r_2b, r_mb); r_bonded_limit = tenPercentMargin*r_bonded; - comm->bBondComm = TRUE; + /* This is the (only) place where we turn on the filtering */ + systemInfo.filterBondedCommunication = true; } else { @@ -2296,7 +2302,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, } real rconstr = 0; - if (dd->splitConstraints && options.constraintCommunicationRange <= 0) + if (systemInfo.haveSplitConstraints && options.constraintCommunicationRange <= 0) { /* There is a cell size limit due to the constraints (P-LINCS) */ rconstr = gmx::constr_r_max(mdlog, mtop, ir); @@ -2357,7 +2363,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, if (ddSetup.numDomains[XX] == 0) { char buf[STRLEN]; - gmx_bool bC = (dd->splitConstraints && rconstr > r_bonded_limit); + gmx_bool bC = (systemInfo.haveSplitConstraints && rconstr > r_bonded_limit); sprintf(buf, "Change the number of ranks or mdrun option %s%s%s", !bC ? "-rdd" : "-rcon", comm->dlbState != DlbState::offUser ? " or -dds" : "", @@ -2477,7 +2483,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, comm->cellsize_limit = systemInfo.cellsizeLimit; if (systemInfo.haveInterDomainBondeds && comm->cutoff_mbody == 0) { - if (comm->bBondComm || !isDlbDisabled(comm)) + if (systemInfo.filterBondedCommunication || !isDlbDisabled(comm)) { /* Set the bonded communication distance to halfway * the minimum and the maximum, @@ -2491,7 +2497,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, comm->cutoff_mbody = std::min(comm->cutoff_mbody, options.dlbScaling*acs); } - if (!comm->bBondComm) + if (!systemInfo.filterBondedCommunication) { /* Without bBondComm do not go beyond the n.b. cut-off */ comm->cutoff_mbody = std::min(comm->cutoff_mbody, systemInfo.cutoff); @@ -2520,7 +2526,8 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, { fprintf(debug, "Bonded atom communication beyond the cut-off: %s\n" "cellsize limit %f\n", - gmx::boolToString(comm->bBondComm), comm->cellsize_limit); + gmx::boolToString(systemInfo.filterBondedCommunication), + comm->cellsize_limit); } if (MASTER(cr)) @@ -2557,7 +2564,7 @@ void dd_init_bondeds(FILE *fplog, comm = dd->comm; - if (comm->bBondComm) + if (comm->systemInfo.filterBondedCommunication) { /* Communicate atoms beyond the cut-off for bonded interactions */ comm = dd->comm; @@ -2646,7 +2653,8 @@ static void writeSettings(gmx::TextWriter *log, if (comm->systemInfo.haveInterDomainBondeds || haveInterDomainVsites || - dd->splitConstraints || dd->splitSettles) + comm->systemInfo.haveSplitConstraints || + comm->systemInfo.haveSplitSettles) { std::string decompUnits; if (comm->systemInfo.useUpdateGroups) @@ -2685,14 +2693,14 @@ static void writeSettings(gmx::TextWriter *log, std::max(comm->systemInfo.cutoff, comm->cutoff_mbody)); log->writeLineFormatted("%40s %-7s %6.3f nm", "multi-body bonded interactions", "(-rdd)", - (comm->bBondComm || isDlbOn(dd->comm)) ? comm->cutoff_mbody : std::min(comm->systemInfo.cutoff, limit)); + (comm->systemInfo.filterBondedCommunication || isDlbOn(dd->comm)) ? comm->cutoff_mbody : std::min(comm->systemInfo.cutoff, limit)); } if (haveInterDomainVsites) { log->writeLineFormatted("%40s %-7s %6.3f nm", "virtual site constructions", "(-rcon)", limit); } - if (dd->splitConstraints || dd->splitSettles) + if (comm->systemInfo.haveSplitConstraints || comm->systemInfo.haveSplitSettles) { std::string separation = gmx::formatString("atoms separated by up to %d constraints", 1+ir->nProjOrder); diff --git a/src/gromacs/domdec/domdec.h b/src/gromacs/domdec/domdec.h index 6f8fa8c8ef..948d6ec556 100644 --- a/src/gromacs/domdec/domdec.h +++ b/src/gromacs/domdec/domdec.h @@ -153,6 +153,9 @@ int dd_pme_maxshift_x(const gmx_domdec_t *dd); /*! \brief Returns the maximum shift for coordinate communication in PME, dim y */ int dd_pme_maxshift_y(const gmx_domdec_t *dd); +/*! \brief Return whether constraints, not including settles, cross domain boundaries */ +bool ddHaveSplitConstraints(const gmx_domdec_t &dd); + /*! \brief Initialized the domain decomposition, chooses the DD grid and PME ranks, return the DD struct */ gmx_domdec_t * init_domain_decomposition(const gmx::MDLogger &mdlog, diff --git a/src/gromacs/domdec/domdec_constraints.cpp b/src/gromacs/domdec/domdec_constraints.cpp index 8954364bb0..5e3d382ca1 100644 --- a/src/gromacs/domdec/domdec_constraints.cpp +++ b/src/gromacs/domdec/domdec_constraints.cpp @@ -69,6 +69,7 @@ #include "gromacs/utility/gmxassert.h" #include "gromacs/utility/smalloc.h" +#include "domdec_internal.h" #include "domdec_specatomcomm.h" /*! \brief Struct used during constraint setup with domain decomposition */ @@ -421,7 +422,9 @@ int dd_make_local_constraints(gmx_domdec_t *dd, int at_start, // This code should not be called unless this condition is true, // because that's the only time init_domdec_constraints is // called... - GMX_RELEASE_ASSERT(dd->splitConstraints || dd->splitSettles, "dd_make_local_constraints called when there are no local constraints"); + GMX_RELEASE_ASSERT(dd->comm->systemInfo.haveSplitConstraints || + dd->comm->systemInfo.haveSplitSettles, + "dd_make_local_constraints called when there are no local constraints"); // ... and init_domdec_constraints always sets // dd->constraint_comm... GMX_RELEASE_ASSERT(dd->constraint_comm, "Invalid use of dd_make_local_constraints before construction of constraint_comm"); @@ -454,7 +457,7 @@ int dd_make_local_constraints(gmx_domdec_t *dd, int at_start, gmx::ArrayRef < const std::vector < int>> at2settle_mt; /* When settle works inside charge groups, we assigned them already */ - if (dd->splitSettles) + if (dd->comm->systemInfo.haveSplitSettles) { // TODO Perhaps gmx_domdec_constraints_t should keep a valid constr? GMX_RELEASE_ASSERT(constr != nullptr, "Must have valid constraints object"); diff --git a/src/gromacs/domdec/domdec_internal.h b/src/gromacs/domdec/domdec_internal.h index 812ab3aa56..bb6c37fc63 100644 --- a/src/gromacs/domdec/domdec_internal.h +++ b/src/gromacs/domdec/domdec_internal.h @@ -456,6 +456,14 @@ struct DDSystemInfo real cutoff = 0; //! The lower limit for the DD cell size real cellsizeLimit = 0; + + //! Can atoms connected by constraints be assigned to different domains? + bool haveSplitConstraints = false; + //! Can atoms connected by settles be assigned to different domains? + bool haveSplitSettles = false; + + //! Whether to only communicate atoms beyond the non-bonded cut-off when they are involved in bonded interactions with non-local atoms + bool filterBondedCommunication = false; }; /*! \brief Struct for domain decomposition communication @@ -509,9 +517,7 @@ struct gmx_domdec_comm_t // NOLINT (clang-analyzer-optin.performance.Padding) //! Centers of mass of local update groups std::unique_ptr updateGroupsCog; - /* Data for the optional bonded interaction atom communication range */ - /**< Only communicate atoms beyond the non-bonded cut-off when they are involved in bonded interactions with non-local atoms */ - gmx_bool bBondComm = false; + /* Data for the optional filtering of communication of atoms for bonded interactions */ /**< Links between cg's through bonded interactions */ t_blocka *cglink = nullptr; /**< Local cg availability, TODO: remove when group scheme is removed */ diff --git a/src/gromacs/domdec/domdec_struct.h b/src/gromacs/domdec/domdec_struct.h index 0d4cd47e95..4a43576e33 100644 --- a/src/gromacs/domdec/domdec_struct.h +++ b/src/gromacs/domdec/domdec_struct.h @@ -189,11 +189,6 @@ struct gmx_domdec_t { //NOLINT(clang-analyzer-optin.performance.Padding) /* Only available on the master node */ std::unique_ptr ma; - /* Can atoms connected by constraints be assigned to different domains? */ - bool splitConstraints = false; - /* Can atoms connected by settles be assigned to different domains? */ - bool splitSettles = false; - /* Global atom number to interaction list */ gmx_reverse_top_t *reverse_top = nullptr; int nbonded_global = 0; diff --git a/src/gromacs/domdec/domdec_topology.cpp b/src/gromacs/domdec/domdec_topology.cpp index 5009a18faf..5acc05c167 100644 --- a/src/gromacs/domdec/domdec_topology.cpp +++ b/src/gromacs/domdec/domdec_topology.cpp @@ -746,7 +746,8 @@ void dd_make_reverse_top(FILE *fplog, dd->reverse_top = new gmx_reverse_top_t; *dd->reverse_top = make_reverse_top(mtop, ir->efep != efepNO, - !dd->splitConstraints, !dd->splitSettles, + !dd->comm->systemInfo.haveSplitConstraints, + !dd->comm->systemInfo.haveSplitSettles, bBCheck, &dd->nbonded_global); gmx_reverse_top_t *rt = dd->reverse_top; @@ -796,7 +797,7 @@ void dd_make_reverse_top(FILE *fplog, init_domdec_vsites(dd, vsite->numInterUpdategroupVsites); } - if (dd->splitConstraints || dd->splitSettles) + if (dd->comm->systemInfo.haveSplitConstraints || dd->comm->systemInfo.haveSplitSettles) { init_domdec_constraints(dd, mtop); } diff --git a/src/gromacs/domdec/partition.cpp b/src/gromacs/domdec/partition.cpp index 15a3a8809c..cf1adc3acf 100644 --- a/src/gromacs/domdec/partition.cpp +++ b/src/gromacs/domdec/partition.cpp @@ -1913,7 +1913,7 @@ static void setup_dd_communication(gmx_domdec_t *dd, comm->dth.resize(numThreads); } - bBondComm = comm->bBondComm; + bBondComm = comm->systemInfo.filterBondedCommunication; /* Do we need to determine extra distances for multi-body bondeds? */ bDistMB = (comm->systemInfo.haveInterDomainMultiBodyBondeds && isDlbOn(dd->comm) && dd->ndim > 1); @@ -3212,7 +3212,7 @@ void dd_partition_system(FILE *fplog, } break; case DDAtomRanges::Type::Constraints: - if (dd->splitConstraints || dd->splitSettles) + if (dd->comm->systemInfo.haveSplitConstraints || dd->comm->systemInfo.haveSplitSettles) { /* Only for inter-cg constraints we need special code */ n = dd_make_local_constraints(dd, n, &top_global, fr->cginfo.data(), diff --git a/src/gromacs/mdlib/constr.cpp b/src/gromacs/mdlib/constr.cpp index 11c7bb7d6d..2be5fd3c3a 100644 --- a/src/gromacs/mdlib/constr.cpp +++ b/src/gromacs/mdlib/constr.cpp @@ -1069,13 +1069,13 @@ Constraints::Impl::Impl(const gmx_mtop_t &mtop_p, { lincsd = init_lincs(log, mtop, nflexcon, at2con_mt, - DOMAINDECOMP(cr) && cr->dd->splitConstraints, + DOMAINDECOMP(cr) && ddHaveSplitConstraints(*cr->dd), ir.nLincsIter, ir.nProjOrder); } if (ir.eConstrAlg == econtSHAKE) { - if (DOMAINDECOMP(cr) && cr->dd->splitConstraints) + if (DOMAINDECOMP(cr) && ddHaveSplitConstraints(*cr->dd)) { gmx_fatal(FARGS, "SHAKE is not supported with domain decomposition and constraint that cross charge group boundaries, use LINCS"); } -- 2.22.0