Move update grouping to DDSystemInfo

Change-Id: Ida28b87a4a0ff004d25ca4d6f0f78f30a19536dd

diff --git a/src/gromacs/domdec/distribute.cpp b/src/gromacs/domdec/distribute.cpp
index 25efe43f9defa737a35290c0c2ef5dc445d56208..e0615aba27ae8e2dcfd56e9215ce0ece1da81243 100644 (file)
--- a/src/gromacs/domdec/distribute.cpp
+++ b/src/gromacs/domdec/distribute.cpp
@@ -408,12 +408,12 @@ getAtomGroupDistribution(const gmx::MDLogger &mdlog,
 
     std::vector < std::vector < int>> indices(dd->nnodes);
 
-    if (dd->comm->useUpdateGroups)
+    if (dd->comm->systemInfo.useUpdateGroups)
     {
         int atomOffset = 0;
         for (const gmx_molblock_t &molblock : mtop.molblock)
         {
-            const auto &updateGrouping = dd->comm->updateGroupingPerMoleculetype[molblock.type];
+            const auto &updateGrouping = dd->comm->systemInfo.updateGroupingPerMoleculetype[molblock.type];
 
             for (int mol = 0; mol < molblock.nmol; mol++)
             {

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index c50134cb8283dcf0c2e25ee4355a95ee149e324c..71dbfb60b5b824f7044d1bce35378d8e106161b1 100644 (file)
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -211,7 +211,7 @@ t_block *dd_charge_groups_global(gmx_domdec_t *dd)
 gmx::ArrayRef<const gmx::RangePartitioning> getUpdateGroupingPerMoleculetype(const gmx_domdec_t &dd)
 {
     GMX_RELEASE_ASSERT(dd.comm, "Need a valid dd.comm");
-    return dd.comm->updateGroupingPerMoleculetype;
+    return dd.comm->systemInfo.updateGroupingPerMoleculetype;
 }
 
 void dd_store_state(gmx_domdec_t *dd, t_state *state)
@@ -2042,9 +2042,8 @@ static bool systemHasConstraintsOrVsites(const gmx_mtop_t &mtop)
 static void setupUpdateGroups(const gmx::MDLogger &mdlog,
                               const gmx_mtop_t    &mtop,
                               const t_inputrec    &inputrec,
-                              real                 cutoffMargin,
-                              int                  numMpiRanksTotal,
-                              gmx_domdec_comm_t   *comm)
+                              const real           cutoffMargin,
+                              DDSystemInfo        *systemInfo)
 {
     /* When we have constraints and/or vsites, it is beneficial to use
      * update groups (when possible) to allow independent update of groups.
@@ -2055,48 +2054,41 @@ static void setupUpdateGroups(const gmx::MDLogger &mdlog,
         return;
     }
 
-    comm->updateGroupingPerMoleculetype = gmx::makeUpdateGroups(mtop);
-    comm->useUpdateGroups               =
-        (!comm->updateGroupingPerMoleculetype.empty() &&
+    systemInfo->updateGroupingPerMoleculetype = gmx::makeUpdateGroups(mtop);
+    systemInfo->useUpdateGroups               =
+        (!systemInfo->updateGroupingPerMoleculetype.empty() &&
          getenv("GMX_NO_UPDATEGROUPS") == nullptr);
 
-    if (comm->useUpdateGroups)
+    if (systemInfo->useUpdateGroups)
     {
         int numUpdateGroups = 0;
         for (const auto &molblock : mtop.molblock)
         {
-            numUpdateGroups += molblock.nmol*comm->updateGroupingPerMoleculetype[molblock.type].numBlocks();
+            numUpdateGroups += molblock.nmol*systemInfo->updateGroupingPerMoleculetype[molblock.type].numBlocks();
         }
 
-        /* Note: We would like to use dd->nnodes for the atom count estimate,
-         *       but that is not yet available here. But this anyhow only
-         *       affect performance up to the second dd_partition_system call.
-         */
-        int homeAtomCountEstimate =  mtop.natoms/numMpiRanksTotal;
-        comm->updateGroupsCog =
-            std::make_unique<gmx::UpdateGroupsCog>(mtop,
-                                                   comm->updateGroupingPerMoleculetype,
-                                                   maxReferenceTemperature(inputrec),
-                                                   homeAtomCountEstimate);
+        systemInfo->maxUpdateGroupRadius =
+            computeMaxUpdateGroupRadius(mtop,
+                                        systemInfo->updateGroupingPerMoleculetype,
+                                        maxReferenceTemperature(inputrec));
 
         /* To use update groups, the large domain-to-domain cutoff distance
          * should be compatible with the box size.
          */
-        comm->useUpdateGroups = (atomToAtomIntoDomainToDomainCutoff(*comm, 0) < cutoffMargin);
+        systemInfo->useUpdateGroups = (atomToAtomIntoDomainToDomainCutoff(*systemInfo, 0) < cutoffMargin);
 
-        if (comm->useUpdateGroups)
+        if (systemInfo->useUpdateGroups)
         {
             GMX_LOG(mdlog.info).appendTextFormatted(
                     "Using update groups, nr %d, average size %.1f atoms, max. radius %.3f nm\n",
                     numUpdateGroups,
                     mtop.natoms/static_cast<double>(numUpdateGroups),
-                    comm->updateGroupsCog->maxUpdateGroupRadius());
+                    systemInfo->maxUpdateGroupRadius);
         }
         else
         {
             GMX_LOG(mdlog.info).appendTextFormatted("The combination of rlist and box size prohibits the use of update groups\n");
-            comm->updateGroupingPerMoleculetype.clear();
-            comm->updateGroupsCog.reset(nullptr);
+            systemInfo->updateGroupingPerMoleculetype.clear();
         }
     }
 }
@@ -2143,22 +2135,37 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
     /* Allocate the charge group/atom sorting struct */
     comm->sort = std::make_unique<gmx_domdec_sort_t>();
 
+    /* Generate the simulation system information */
+    DDSystemInfo &systemInfo = comm->systemInfo;
+
     /* We need to decide on update groups early, as this affects communication distances */
-    comm->useUpdateGroups = false;
+    systemInfo.useUpdateGroups = false;
     if (ir->cutoff_scheme == ecutsVERLET)
     {
         real cutoffMargin = std::sqrt(max_cutoff2(ir->ePBC, box)) - ir->rlist;
-        setupUpdateGroups(mdlog, *mtop, *ir, cutoffMargin, cr->nnodes, comm);
-    }
+        setupUpdateGroups(mdlog, *mtop, *ir, cutoffMargin, &systemInfo);
 
-    DDSystemInfo &systemInfo = comm->systemInfo;
+        if (systemInfo.useUpdateGroups)
+        {
+            /* Note: We would like to use dd->nnodes for the atom count estimate,
+             *       but that is not yet available here. But this anyhow only
+             *       affect performance up to the second dd_partition_system call.
+             */
+            const int homeAtomCountEstimate =  mtop->natoms/cr->nnodes;
+            comm->updateGroupsCog =
+                std::make_unique<gmx::UpdateGroupsCog>(*mtop,
+                                                       systemInfo.updateGroupingPerMoleculetype,
+                                                       maxReferenceTemperature(*ir),
+                                                       homeAtomCountEstimate);
+        }
+    }
 
     // TODO: Check whether all bondeds are within update groups
     systemInfo.haveInterDomainBondeds          = (mtop->natoms > gmx_mtop_num_molecules(*mtop) ||
                                                   mtop->bIntermolecularInteractions);
     systemInfo.haveInterDomainMultiBodyBondeds = (multi_body_bondeds_count(mtop) > 0);
 
-    if (comm->useUpdateGroups)
+    if (systemInfo.useUpdateGroups)
     {
         dd->splitConstraints = false;
         dd->splitSettles     = false;
@@ -2179,7 +2186,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
     }
     else
     {
-        systemInfo.cutoff = atomToAtomIntoDomainToDomainCutoff(*comm, ir->rlist);
+        systemInfo.cutoff = atomToAtomIntoDomainToDomainCutoff(systemInfo, ir->rlist);
     }
     systemInfo.minCutoffForMultiBody = 0;
 
@@ -2197,7 +2204,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
     constexpr real c_chanceThatAtomMovesBeyondDomain = 1e-12;
     const real     limitForAtomDisplacement          =
         minCellSizeForAtomDisplacement(*mtop, *ir,
-                                       comm->updateGroupingPerMoleculetype,
+                                       systemInfo.updateGroupingPerMoleculetype,
                                        c_chanceThatAtomMovesBeyondDomain);
     GMX_LOG(mdlog.info).appendTextFormatted(
             "Minimum cell size due to atom displacement: %.3f nm",
@@ -2220,7 +2227,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
         if (options.minimumCommunicationRange > 0)
         {
             systemInfo.minCutoffForMultiBody =
-                atomToAtomIntoDomainToDomainCutoff(*comm, options.minimumCommunicationRange);
+                atomToAtomIntoDomainToDomainCutoff(systemInfo, options.minimumCommunicationRange);
             if (options.useBondedCommunication)
             {
                 comm->bBondComm = (systemInfo.minCutoffForMultiBody > systemInfo.cutoff);
@@ -2635,14 +2642,14 @@ static void writeSettings(gmx::TextWriter       *log,
     }
 
     const bool haveInterDomainVsites =
-        (countInterUpdategroupVsites(*mtop, comm->updateGroupingPerMoleculetype) != 0);
+        (countInterUpdategroupVsites(*mtop, comm->systemInfo.updateGroupingPerMoleculetype) != 0);
 
     if (comm->systemInfo.haveInterDomainBondeds ||
         haveInterDomainVsites ||
         dd->splitConstraints || dd->splitSettles)
     {
         std::string decompUnits;
-        if (comm->useUpdateGroups)
+        if (comm->systemInfo.useUpdateGroups)
         {
             decompUnits = "atom groups";
         }

diff --git a/src/gromacs/domdec/domdec_internal.h b/src/gromacs/domdec/domdec_internal.h
index 3c03e7b25ff756c9f4c777cf38244f7cd3af6f1b..812ab3aa567fa28228891780e438745c0a75bf1c 100644 (file)
--- a/src/gromacs/domdec/domdec_internal.h
+++ b/src/gromacs/domdec/domdec_internal.h
@@ -438,6 +438,13 @@ struct dd_comm_setup_work_t
 /*! \brief Information about the simulated system */
 struct DDSystemInfo
 {
+    //! True when update groups are used
+    bool                                useUpdateGroups = false;
+    //! Update atom grouping for each molecule type
+    std::vector<gmx::RangePartitioning> updateGroupingPerMoleculetype;
+    //! The maximum radius over all update groups
+    real                                maxUpdateGroupRadius;
+
     //! Are there inter-domain bonded interactions?
     bool haveInterDomainBondeds          = false;
     //! Are there inter-domain multi-body interactions?
@@ -499,10 +506,6 @@ struct gmx_domdec_comm_t // NOLINT (clang-analyzer-optin.performance.Padding)
     /**< Data structure for cg/atom sorting */
     std::unique_ptr<gmx_domdec_sort_t> sort;
 
-    //! True when update groups are used
-    bool                                  useUpdateGroups = false;
-    //!  Update atom grouping for each molecule type
-    std::vector<gmx::RangePartitioning>   updateGroupingPerMoleculetype;
     //! Centers of mass of local update groups
     std::unique_ptr<gmx::UpdateGroupsCog> updateGroupsCog;
 

diff --git a/src/gromacs/domdec/partition.cpp b/src/gromacs/domdec/partition.cpp
index 6cf09729848b87ef1f2022d641f08ec156e17eef..15a3a8809c18e6c796bf471658b8315d25cf5846 100644 (file)
--- a/src/gromacs/domdec/partition.cpp
+++ b/src/gromacs/domdec/partition.cpp
@@ -1921,8 +1921,8 @@ static void setup_dd_communication(gmx_domdec_t *dd,
     /* Do we need to determine extra distances for only two-body bondeds? */
     bDist2B = (bBondComm && !bDistMB);
 
-    const real r_comm2  = gmx::square(domainToDomainIntoAtomToDomainCutoff(*comm, comm->systemInfo.cutoff));
-    const real r_bcomm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(*comm, comm->cutoff_mbody));
+    const real r_comm2  = gmx::square(domainToDomainIntoAtomToDomainCutoff(comm->systemInfo, comm->systemInfo.cutoff));
+    const real r_bcomm2 = gmx::square(domainToDomainIntoAtomToDomainCutoff(comm->systemInfo, comm->cutoff_mbody));
 
     if (debug)
     {
@@ -3042,7 +3042,7 @@ void dd_partition_system(FILE                    *fplog,
         write_dd_grid_pdb("dd_grid", step, dd, state_local->box, &ddbox);
     }
 
-    if (comm->useUpdateGroups)
+    if (comm->systemInfo.useUpdateGroups)
     {
         comm->updateGroupsCog->addCogs(gmx::arrayRefFromArray(dd->globalAtomGroupIndices.data(), dd->ncg_home),
                                        state_local->x);
@@ -3068,7 +3068,7 @@ void dd_partition_system(FILE                    *fplog,
 
         GMX_RELEASE_ASSERT(bSortCG, "Sorting is required after redistribution");
 
-        if (comm->useUpdateGroups)
+        if (comm->systemInfo.useUpdateGroups)
         {
             comm->updateGroupsCog->addCogs(gmx::arrayRefFromArray(dd->globalAtomGroupIndices.data(), dd->ncg_home),
                                            state_local->x);
@@ -3147,7 +3147,7 @@ void dd_partition_system(FILE                    *fplog,
         }
     }
 
-    if (comm->useUpdateGroups)
+    if (comm->systemInfo.useUpdateGroups)
     {
         /* The update groups cog's are invalid after sorting
          * and need to be cleared before the next partitioning anyhow.

diff --git a/src/gromacs/domdec/redistribute.cpp b/src/gromacs/domdec/redistribute.cpp
index d947b034b9e68baaf0942d10a9702e22acbf6433..bb73ed56ad47cd41a938fab8eac9e7be785ceeec 100644 (file)
--- a/src/gromacs/domdec/redistribute.cpp
+++ b/src/gromacs/domdec/redistribute.cpp
@@ -117,7 +117,7 @@ copyMovedUpdateGroupCogs(gmx::ArrayRef<const int> move,
         if (m >= 0)
         {
             /* Copy to the communication buffer */
-            const gmx::RVec &cog = (comm->useUpdateGroups ?
+            const gmx::RVec &cog = (comm->systemInfo.useUpdateGroups ?
                                     comm->updateGroupsCog->cogForAtom(g) :
                                     coordinates[g]);
             copy_rvec(cog, comm->cgcm_state[m][pos_vec[m]]);
@@ -163,7 +163,7 @@ static void print_cg_move(FILE *fplog,
 
     fprintf(fplog, "\nStep %" PRId64 ":\n", step);
 
-    if (comm->useUpdateGroups)
+    if (comm->systemInfo.useUpdateGroups)
     {
         mesg += "The update group starting at atom";
     }
@@ -613,7 +613,7 @@ void dd_redistribute_cg(FILE *fplog, int64_t step,
     /* Compute the center of geometry for all home charge groups
      * and put them in the box and determine where they should go.
      */
-    std::vector<PbcAndFlag>  pbcAndFlags(comm->useUpdateGroups ? comm->updateGroupsCog->numCogs() : 0);
+    std::vector<PbcAndFlag>  pbcAndFlags(comm->systemInfo.useUpdateGroups ? comm->updateGroupsCog->numCogs() : 0);
 
 #pragma omp parallel num_threads(nthread)
     {
@@ -621,7 +621,7 @@ void dd_redistribute_cg(FILE *fplog, int64_t step,
         {
             const int thread = gmx_omp_get_thread_num();
 
-            if (comm->useUpdateGroups)
+            if (comm->systemInfo.useUpdateGroups)
             {
                 const auto &updateGroupsCog = *comm->updateGroupsCog;
                 const int   numGroups       = updateGroupsCog.numCogs();

diff --git a/src/gromacs/domdec/utility.h b/src/gromacs/domdec/utility.h
index e4d2519f876018f84b70570046c79d2eea98dc9f..51f3f34f282988aca906dacfd011e25afbbaea64 100644 (file)
--- a/src/gromacs/domdec/utility.h
+++ b/src/gromacs/domdec/utility.h
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2018, by the GROMACS development team, led by
+ * Copyright (c) 2018,2019, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -102,14 +102,12 @@ void dd_check_alloc_ncg(t_forcerec              *fr,
 
 /*! \brief Returns a domain-to-domain cutoff distance given an atom-to-atom cutoff */
 static inline real
-atomToAtomIntoDomainToDomainCutoff(const gmx_domdec_comm_t &comm,
-                                   real                     cutoff)
+atomToAtomIntoDomainToDomainCutoff(const DDSystemInfo &systemInfo,
+                                   real                cutoff)
 {
-    if (comm.useUpdateGroups)
+    if (systemInfo.useUpdateGroups)
     {
-        GMX_ASSERT(comm.updateGroupsCog, "updateGroupsCog should be initialized here");
-
-        cutoff += 2*comm.updateGroupsCog->maxUpdateGroupRadius();
+        cutoff += 2*systemInfo.maxUpdateGroupRadius;
     }
 
     return cutoff;
@@ -117,14 +115,12 @@ atomToAtomIntoDomainToDomainCutoff(const gmx_domdec_comm_t &comm,
 
 /*! \brief Returns an atom-to-domain cutoff distance given a domain-to-domain cutoff */
 static inline real
-domainToDomainIntoAtomToDomainCutoff(const gmx_domdec_comm_t &comm,
-                                     real                     cutoff)
+domainToDomainIntoAtomToDomainCutoff(const DDSystemInfo &systemInfo,
+                                     real                cutoff)
 {
-    if (comm.useUpdateGroups)
+    if (systemInfo.useUpdateGroups)
     {
-        GMX_ASSERT(comm.updateGroupsCog, "updateGroupsCog should be initialized here");
-
-        cutoff -= comm.updateGroupsCog->maxUpdateGroupRadius();
+        cutoff -= systemInfo.maxUpdateGroupRadius;
     }
 
     return cutoff;