From dde2ecaac6c61438027b05588b672f8da613f77a Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Tue, 3 Sep 2019 22:43:34 +0200 Subject: [PATCH] Move update grouping to DDSystemInfo Change-Id: Ida28b87a4a0ff004d25ca4d6f0f78f30a19536dd --- src/gromacs/domdec/distribute.cpp | 4 +- src/gromacs/domdec/domdec.cpp | 75 +++++++++++++++------------- src/gromacs/domdec/domdec_internal.h | 11 ++-- src/gromacs/domdec/partition.cpp | 10 ++-- src/gromacs/domdec/redistribute.cpp | 8 +-- src/gromacs/domdec/utility.h | 22 ++++---- 6 files changed, 68 insertions(+), 62 deletions(-) diff --git a/src/gromacs/domdec/distribute.cpp b/src/gromacs/domdec/distribute.cpp index 25efe43f9d..e0615aba27 100644 --- a/src/gromacs/domdec/distribute.cpp +++ b/src/gromacs/domdec/distribute.cpp @@ -408,12 +408,12 @@ getAtomGroupDistribution(const gmx::MDLogger &mdlog, std::vector < std::vector < int>> indices(dd->nnodes); - if (dd->comm->useUpdateGroups) + if (dd->comm->systemInfo.useUpdateGroups) { int atomOffset = 0; for (const gmx_molblock_t &molblock : mtop.molblock) { - const auto &updateGrouping = dd->comm->updateGroupingPerMoleculetype[molblock.type]; + const auto &updateGrouping = dd->comm->systemInfo.updateGroupingPerMoleculetype[molblock.type]; for (int mol = 0; mol < molblock.nmol; mol++) { diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index c50134cb82..71dbfb60b5 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -211,7 +211,7 @@ t_block *dd_charge_groups_global(gmx_domdec_t *dd) gmx::ArrayRef getUpdateGroupingPerMoleculetype(const gmx_domdec_t &dd) { GMX_RELEASE_ASSERT(dd.comm, "Need a valid dd.comm"); - return dd.comm->updateGroupingPerMoleculetype; + return dd.comm->systemInfo.updateGroupingPerMoleculetype; } void dd_store_state(gmx_domdec_t *dd, t_state *state) @@ -2042,9 +2042,8 @@ static bool systemHasConstraintsOrVsites(const gmx_mtop_t &mtop) static void setupUpdateGroups(const gmx::MDLogger &mdlog, const gmx_mtop_t &mtop, const t_inputrec &inputrec, - real cutoffMargin, - int numMpiRanksTotal, - gmx_domdec_comm_t *comm) + const real cutoffMargin, + DDSystemInfo *systemInfo) { /* When we have constraints and/or vsites, it is beneficial to use * update groups (when possible) to allow independent update of groups. @@ -2055,48 +2054,41 @@ static void setupUpdateGroups(const gmx::MDLogger &mdlog, return; } - comm->updateGroupingPerMoleculetype = gmx::makeUpdateGroups(mtop); - comm->useUpdateGroups = - (!comm->updateGroupingPerMoleculetype.empty() && + systemInfo->updateGroupingPerMoleculetype = gmx::makeUpdateGroups(mtop); + systemInfo->useUpdateGroups = + (!systemInfo->updateGroupingPerMoleculetype.empty() && getenv("GMX_NO_UPDATEGROUPS") == nullptr); - if (comm->useUpdateGroups) + if (systemInfo->useUpdateGroups) { int numUpdateGroups = 0; for (const auto &molblock : mtop.molblock) { - numUpdateGroups += molblock.nmol*comm->updateGroupingPerMoleculetype[molblock.type].numBlocks(); + numUpdateGroups += molblock.nmol*systemInfo->updateGroupingPerMoleculetype[molblock.type].numBlocks(); } - /* Note: We would like to use dd->nnodes for the atom count estimate, - * but that is not yet available here. But this anyhow only - * affect performance up to the second dd_partition_system call. - */ - int homeAtomCountEstimate = mtop.natoms/numMpiRanksTotal; - comm->updateGroupsCog = - std::make_unique(mtop, - comm->updateGroupingPerMoleculetype, - maxReferenceTemperature(inputrec), - homeAtomCountEstimate); + systemInfo->maxUpdateGroupRadius = + computeMaxUpdateGroupRadius(mtop, + systemInfo->updateGroupingPerMoleculetype, + maxReferenceTemperature(inputrec)); /* To use update groups, the large domain-to-domain cutoff distance * should be compatible with the box size. */ - comm->useUpdateGroups = (atomToAtomIntoDomainToDomainCutoff(*comm, 0) < cutoffMargin); + systemInfo->useUpdateGroups = (atomToAtomIntoDomainToDomainCutoff(*systemInfo, 0) < cutoffMargin); - if (comm->useUpdateGroups) + if (systemInfo->useUpdateGroups) { GMX_LOG(mdlog.info).appendTextFormatted( "Using update groups, nr %d, average size %.1f atoms, max. radius %.3f nm\n", numUpdateGroups, mtop.natoms/static_cast(numUpdateGroups), - comm->updateGroupsCog->maxUpdateGroupRadius()); + systemInfo->maxUpdateGroupRadius); } else { GMX_LOG(mdlog.info).appendTextFormatted("The combination of rlist and box size prohibits the use of update groups\n"); - comm->updateGroupingPerMoleculetype.clear(); - comm->updateGroupsCog.reset(nullptr); + systemInfo->updateGroupingPerMoleculetype.clear(); } } } @@ -2143,22 +2135,37 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, /* Allocate the charge group/atom sorting struct */ comm->sort = std::make_unique(); + /* Generate the simulation system information */ + DDSystemInfo &systemInfo = comm->systemInfo; + /* We need to decide on update groups early, as this affects communication distances */ - comm->useUpdateGroups = false; + systemInfo.useUpdateGroups = false; if (ir->cutoff_scheme == ecutsVERLET) { real cutoffMargin = std::sqrt(max_cutoff2(ir->ePBC, box)) - ir->rlist; - setupUpdateGroups(mdlog, *mtop, *ir, cutoffMargin, cr->nnodes, comm); - } + setupUpdateGroups(mdlog, *mtop, *ir, cutoffMargin, &systemInfo); - DDSystemInfo &systemInfo = comm->systemInfo; + if (systemInfo.useUpdateGroups) + { + /* Note: We would like to use dd->nnodes for the atom count estimate, + * but that is not yet available here. But this anyhow only + * affect performance up to the second dd_partition_system call. + */ + const int homeAtomCountEstimate = mtop->natoms/cr->nnodes; + comm->updateGroupsCog = + std::make_unique(*mtop, + systemInfo.updateGroupingPerMoleculetype, + maxReferenceTemperature(*ir), + homeAtomCountEstimate); + } + } // TODO: Check whether all bondeds are within update groups systemInfo.haveInterDomainBondeds = (mtop->natoms > gmx_mtop_num_molecules(*mtop) || mtop->bIntermolecularInteractions); systemInfo.haveInterDomainMultiBodyBondeds = (multi_body_bondeds_count(mtop) > 0); - if (comm->useUpdateGroups) + if (systemInfo.useUpdateGroups) { dd->splitConstraints = false; dd->splitSettles = false; @@ -2179,7 +2186,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, } else { - systemInfo.cutoff = atomToAtomIntoDomainToDomainCutoff(*comm, ir->rlist); + systemInfo.cutoff = atomToAtomIntoDomainToDomainCutoff(systemInfo, ir->rlist); } systemInfo.minCutoffForMultiBody = 0; @@ -2197,7 +2204,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, constexpr real c_chanceThatAtomMovesBeyondDomain = 1e-12; const real limitForAtomDisplacement = minCellSizeForAtomDisplacement(*mtop, *ir, - comm->updateGroupingPerMoleculetype, + systemInfo.updateGroupingPerMoleculetype, c_chanceThatAtomMovesBeyondDomain); GMX_LOG(mdlog.info).appendTextFormatted( "Minimum cell size due to atom displacement: %.3f nm", @@ -2220,7 +2227,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog, if (options.minimumCommunicationRange > 0) { systemInfo.minCutoffForMultiBody = - atomToAtomIntoDomainToDomainCutoff(*comm, options.minimumCommunicationRange); + atomToAtomIntoDomainToDomainCutoff(systemInfo, options.minimumCommunicationRange); if (options.useBondedCommunication) { comm->bBondComm = (systemInfo.minCutoffForMultiBody > systemInfo.cutoff); @@ -2635,14 +2642,14 @@ static void writeSettings(gmx::TextWriter *log, } const bool haveInterDomainVsites = - (countInterUpdategroupVsites(*mtop, comm->updateGroupingPerMoleculetype) != 0); + (countInterUpdategroupVsites(*mtop, comm->systemInfo.updateGroupingPerMoleculetype) != 0); if (comm->systemInfo.haveInterDomainBondeds || haveInterDomainVsites || dd->splitConstraints || dd->splitSettles) { std::string decompUnits; - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { decompUnits = "atom groups"; } diff --git a/src/gromacs/domdec/domdec_internal.h b/src/gromacs/domdec/domdec_internal.h index 3c03e7b25f..812ab3aa56 100644 --- a/src/gromacs/domdec/domdec_internal.h +++ b/src/gromacs/domdec/domdec_internal.h @@ -438,6 +438,13 @@ struct dd_comm_setup_work_t /*! \brief Information about the simulated system */ struct DDSystemInfo { + //! True when update groups are used + bool useUpdateGroups = false; + //! Update atom grouping for each molecule type + std::vector updateGroupingPerMoleculetype; + //! The maximum radius over all update groups + real maxUpdateGroupRadius; + //! Are there inter-domain bonded interactions? bool haveInterDomainBondeds = false; //! Are there inter-domain multi-body interactions? @@ -499,10 +506,6 @@ struct gmx_domdec_comm_t // NOLINT (clang-analyzer-optin.performance.Padding) /**< Data structure for cg/atom sorting */ std::unique_ptr sort; - //! True when update groups are used - bool useUpdateGroups = false; - //! Update atom grouping for each molecule type - std::vector updateGroupingPerMoleculetype; //! Centers of mass of local update groups std::unique_ptr updateGroupsCog; diff --git a/src/gromacs/domdec/partition.cpp b/src/gromacs/domdec/partition.cpp index 6cf0972984..15a3a8809c 100644 --- a/src/gromacs/domdec/partition.cpp +++ b/src/gromacs/domdec/partition.cpp @@ -1921,8 +1921,8 @@ static void setup_dd_communication(gmx_domdec_t *dd, /* Do we need to determine extra distances for only two-body bondeds? */ bDist2B = (bBondComm && !bDistMB); - const real r_comm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(*comm, comm->systemInfo.cutoff)); - const real r_bcomm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(*comm, comm->cutoff_mbody)); + const real r_comm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(comm->systemInfo, comm->systemInfo.cutoff)); + const real r_bcomm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(comm->systemInfo, comm->cutoff_mbody)); if (debug) { @@ -3042,7 +3042,7 @@ void dd_partition_system(FILE *fplog, write_dd_grid_pdb("dd_grid", step, dd, state_local->box, &ddbox); } - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { comm->updateGroupsCog->addCogs(gmx::arrayRefFromArray(dd->globalAtomGroupIndices.data(), dd->ncg_home), state_local->x); @@ -3068,7 +3068,7 @@ void dd_partition_system(FILE *fplog, GMX_RELEASE_ASSERT(bSortCG, "Sorting is required after redistribution"); - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { comm->updateGroupsCog->addCogs(gmx::arrayRefFromArray(dd->globalAtomGroupIndices.data(), dd->ncg_home), state_local->x); @@ -3147,7 +3147,7 @@ void dd_partition_system(FILE *fplog, } } - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { /* The update groups cog's are invalid after sorting * and need to be cleared before the next partitioning anyhow. diff --git a/src/gromacs/domdec/redistribute.cpp b/src/gromacs/domdec/redistribute.cpp index d947b034b9..bb73ed56ad 100644 --- a/src/gromacs/domdec/redistribute.cpp +++ b/src/gromacs/domdec/redistribute.cpp @@ -117,7 +117,7 @@ copyMovedUpdateGroupCogs(gmx::ArrayRef move, if (m >= 0) { /* Copy to the communication buffer */ - const gmx::RVec &cog = (comm->useUpdateGroups ? + const gmx::RVec &cog = (comm->systemInfo.useUpdateGroups ? comm->updateGroupsCog->cogForAtom(g) : coordinates[g]); copy_rvec(cog, comm->cgcm_state[m][pos_vec[m]]); @@ -163,7 +163,7 @@ static void print_cg_move(FILE *fplog, fprintf(fplog, "\nStep %" PRId64 ":\n", step); - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { mesg += "The update group starting at atom"; } @@ -613,7 +613,7 @@ void dd_redistribute_cg(FILE *fplog, int64_t step, /* Compute the center of geometry for all home charge groups * and put them in the box and determine where they should go. */ - std::vector pbcAndFlags(comm->useUpdateGroups ? comm->updateGroupsCog->numCogs() : 0); + std::vector pbcAndFlags(comm->systemInfo.useUpdateGroups ? comm->updateGroupsCog->numCogs() : 0); #pragma omp parallel num_threads(nthread) { @@ -621,7 +621,7 @@ void dd_redistribute_cg(FILE *fplog, int64_t step, { const int thread = gmx_omp_get_thread_num(); - if (comm->useUpdateGroups) + if (comm->systemInfo.useUpdateGroups) { const auto &updateGroupsCog = *comm->updateGroupsCog; const int numGroups = updateGroupsCog.numCogs(); diff --git a/src/gromacs/domdec/utility.h b/src/gromacs/domdec/utility.h index e4d2519f87..51f3f34f28 100644 --- a/src/gromacs/domdec/utility.h +++ b/src/gromacs/domdec/utility.h @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2018, by the GROMACS development team, led by + * Copyright (c) 2018,2019, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -102,14 +102,12 @@ void dd_check_alloc_ncg(t_forcerec *fr, /*! \brief Returns a domain-to-domain cutoff distance given an atom-to-atom cutoff */ static inline real -atomToAtomIntoDomainToDomainCutoff(const gmx_domdec_comm_t &comm, - real cutoff) +atomToAtomIntoDomainToDomainCutoff(const DDSystemInfo &systemInfo, + real cutoff) { - if (comm.useUpdateGroups) + if (systemInfo.useUpdateGroups) { - GMX_ASSERT(comm.updateGroupsCog, "updateGroupsCog should be initialized here"); - - cutoff += 2*comm.updateGroupsCog->maxUpdateGroupRadius(); + cutoff += 2*systemInfo.maxUpdateGroupRadius; } return cutoff; @@ -117,14 +115,12 @@ atomToAtomIntoDomainToDomainCutoff(const gmx_domdec_comm_t &comm, /*! \brief Returns an atom-to-domain cutoff distance given a domain-to-domain cutoff */ static inline real -domainToDomainIntoAtomToDomainCutoff(const gmx_domdec_comm_t &comm, - real cutoff) +domainToDomainIntoAtomToDomainCutoff(const DDSystemInfo &systemInfo, + real cutoff) { - if (comm.useUpdateGroups) + if (systemInfo.useUpdateGroups) { - GMX_ASSERT(comm.updateGroupsCog, "updateGroupsCog should be initialized here"); - - cutoff -= comm.updateGroupsCog->maxUpdateGroupRadius(); + cutoff -= systemInfo.maxUpdateGroupRadius; } return cutoff; -- 2.22.0