Split off getting of the DD setup

Change-Id: I19e956b04a14df75e91b721954bba52abe49d1d8

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index 05d365bd9d3fb79458e18df9f253686aa39cebe4..382893f2cef48493bdd6a52f767a0af6c407696c 100644 (file)
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -2294,59 +2294,24 @@ getSystemInfo(const gmx::MDLogger           &mdlog,
     return systemInfo;
 }
 
-/*! \brief Set the cell size and interaction limits, as well as the DD grid */
-static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
-                                   t_commrec *cr, gmx_domdec_t *dd,
-                                   const DomdecOptions &options,
-                                   const DDSettings &ddSettings,
-                                   const gmx_mtop_t *mtop,
-                                   const t_inputrec *ir,
-                                   const matrix box,
-                                   gmx::ArrayRef<const gmx::RVec> xGlobal,
-                                   gmx_ddbox_t *ddbox)
-{
-    gmx_domdec_comm_t *comm = dd->comm;
-    comm->ddSettings        = ddSettings;
-
-    /* Initialize to GPU share count to 0, might change later */
-    comm->nrank_gpu_shared = 0;
-
-    comm->dlbState         = comm->ddSettings.initialDlbState;
-    dd_dlb_set_should_check_whether_to_turn_dlb_on(dd, TRUE);
-    /* To consider turning DLB on after 2*nstlist steps we need to check
-     * at partitioning count 3. Thus we need to increase the first count by 2.
-     */
-    comm->ddPartioningCountFirstDlbOff += 2;
-
-    GMX_LOG(mdlog.info).appendTextFormatted(
-            "Dynamic load balancing: %s", edlbs_names[int(comm->dlbState)]);
-
-    comm->bPMELoadBalDLBLimits = FALSE;
-
-    /* Allocate the charge group/atom sorting struct */
-    comm->sort = std::make_unique<gmx_domdec_sort_t>();
-
-    /* Generate the simulation system information */
-    comm->systemInfo               = getSystemInfo(mdlog, cr, options, mtop, ir, box, xGlobal);
-    const DDSystemInfo &systemInfo = comm->systemInfo;
-
-    if (systemInfo.useUpdateGroups)
-    {
-        /* Note: We would like to use dd->nnodes for the atom count estimate,
-         *       but that is not yet available here. But this anyhow only
-         *       affect performance up to the second dd_partition_system call.
-         */
-        const int homeAtomCountEstimate =  mtop->natoms/cr->nnodes;
-        comm->updateGroupsCog =
-            std::make_unique<gmx::UpdateGroupsCog>(*mtop,
-                                                   systemInfo.updateGroupingPerMoleculetype,
-                                                   maxReferenceTemperature(*ir),
-                                                   homeAtomCountEstimate);
-    }
-
-    comm->cgs_gl = gmx_mtop_global_cgs(mtop);
+/*! \brief Set the cell size and interaction limits, as well as the DD grid
+ *
+ * Also computes the initial ddbox.
+ */
+static DDSetup
+getDDSetup(const gmx::MDLogger           &mdlog,
+           t_commrec                     *cr,
+           const DomdecOptions           &options,
+           const DDSettings              &ddSettings,
+           const DDSystemInfo            &systemInfo,
+           const gmx_mtop_t              *mtop,
+           const t_inputrec              *ir,
+           const matrix                   box,
+           gmx::ArrayRef<const gmx::RVec> xGlobal,
+           gmx_ddbox_t                   *ddbox)
+{
+    DDSetup     ddSetup;
 
-    DDSetup ddSetup;
     if (options.numCells[XX] > 0)
     {
         copy_ivec(options.numCells, ddSetup.numDomains);
@@ -2373,7 +2338,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
         ddSetup =
             dd_choose_grid(mdlog, cr, ir, mtop, box, ddbox,
                            options.numPmeRanks,
-                           !isDlbDisabled(comm),
+                           !isDlbDisabled(ddSettings.initialDlbState),
                            options.dlbScaling,
                            systemInfo);
 
@@ -2384,7 +2349,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
                            systemInfo.constraintCommunicationRange > systemInfo.minCutoffForMultiBody);
             sprintf(buf, "Change the number of ranks or mdrun option %s%s%s",
                     !bC ? "-rdd" : "-rcon",
-                    comm->dlbState != DlbState::offUser ? " or -dds" : "",
+                    ddSettings.initialDlbState != DlbState::offUser ? " or -dds" : "",
                     bC ? " or your LINCS settings" : "");
 
             gmx_fatal_collective(FARGS, cr->mpi_comm_mysim, MASTER(cr),
@@ -2412,6 +2377,56 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
         }
     }
 
+    return ddSetup;
+}
+
+/*! \brief Set the cell size and interaction limits, as well as the DD grid */
+static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
+                                   t_commrec *cr, gmx_domdec_t *dd,
+                                   const DomdecOptions &options,
+                                   const DDSettings &ddSettings,
+                                   const DDSystemInfo &systemInfo,
+                                   const DDSetup &ddSetup,
+                                   const gmx_mtop_t *mtop,
+                                   const t_inputrec *ir,
+                                   const gmx_ddbox_t &ddbox)
+{
+    gmx_domdec_comm_t *comm = dd->comm;
+    comm->ddSettings        = ddSettings;
+
+    /* Initialize to GPU share count to 0, might change later */
+    comm->nrank_gpu_shared = 0;
+
+    comm->dlbState         = comm->ddSettings.initialDlbState;
+    dd_dlb_set_should_check_whether_to_turn_dlb_on(dd, TRUE);
+    /* To consider turning DLB on after 2*nstlist steps we need to check
+     * at partitioning count 3. Thus we need to increase the first count by 2.
+     */
+    comm->ddPartioningCountFirstDlbOff += 2;
+
+    comm->bPMELoadBalDLBLimits          = FALSE;
+
+    /* Allocate the charge group/atom sorting struct */
+    comm->sort = std::make_unique<gmx_domdec_sort_t>();
+
+    comm->systemInfo = systemInfo;
+
+    if (systemInfo.useUpdateGroups)
+    {
+        /* Note: We would like to use dd->nnodes for the atom count estimate,
+         *       but that is not yet available here. But this anyhow only
+         *       affect performance up to the second dd_partition_system call.
+         */
+        const int homeAtomCountEstimate =  mtop->natoms/cr->nnodes;
+        comm->updateGroupsCog =
+            std::make_unique<gmx::UpdateGroupsCog>(*mtop,
+                                                   systemInfo.updateGroupingPerMoleculetype,
+                                                   maxReferenceTemperature(*ir),
+                                                   homeAtomCountEstimate);
+    }
+
+    comm->cgs_gl = gmx_mtop_global_cgs(mtop);
+
     /* Set the DD setup given by ddSetup */
     cr->npmenodes = ddSetup.numPmeRanks;
     copy_ivec(ddSetup.numDomains, dd->nc);
@@ -2507,7 +2522,7 @@ static void set_dd_limits_and_grid(const gmx::MDLogger &mdlog,
              * the minimum and the maximum,
              * since the extra communication cost is nearly zero.
              */
-            real acs           = average_cellsize_min(*ddbox, dd->nc);
+            real acs           = average_cellsize_min(ddbox, dd->nc);
             comm->cutoff_mbody = 0.5*(systemInfo.minCutoffForMultiBody + acs);
             if (!isDlbDisabled(comm))
             {
@@ -2961,6 +2976,8 @@ getDDSettings(const gmx::MDLogger     &mdlog,
 
     ddSettings.initialDlbState =
         determineInitialDlbState(mdlog, options.dlbOption, ddSettings.recordLoad, mdrunOptions, &ir);
+    GMX_LOG(mdlog.info).appendTextFormatted("Dynamic load balancing: %s",
+                                            edlbs_names[static_cast<int>(ddSettings.initialDlbState)]);
 
     return ddSettings;
 }
@@ -2985,10 +3002,6 @@ gmx_domdec_t *init_domain_decomposition(const gmx::MDLogger           &mdlog,
     GMX_LOG(mdlog.info).appendTextFormatted(
             "\nInitializing Domain Decomposition on %d ranks", cr->nnodes);
 
-    dd = new gmx_domdec_t(*ir);
-
-    dd->comm = init_dd_comm();
-
     DDSettings  ddSettings = getDDSettings(mdlog, options, mdrunOptions, *ir);
     if (ddSettings.eFlop > 1)
     {
@@ -2996,12 +3009,20 @@ gmx_domdec_t *init_domain_decomposition(const gmx::MDLogger           &mdlog,
         srand(1 + cr->nodeid);
     }
 
-    gmx_ddbox_t ddbox = {0};
+    DDSystemInfo systemInfo = getSystemInfo(mdlog, cr, options, mtop, ir, box, xGlobal);
+
+    gmx_ddbox_t  ddbox      = {0};
+    DDSetup      ddSetup    = getDDSetup(mdlog, cr, options, ddSettings, systemInfo,
+                                         mtop, ir, box, xGlobal, &ddbox);
+
+    dd = new gmx_domdec_t(*ir);
+
+    dd->comm = init_dd_comm();
+
     set_dd_limits_and_grid(mdlog, cr, dd, options,
-                           ddSettings,
+                           ddSettings, systemInfo, ddSetup,
                            mtop, ir,
-                           box, xGlobal,
-                           &ddbox);
+                           ddbox);
 
     make_dd_communicators(mdlog, cr, dd, options.rankOrder);
 

diff --git a/src/gromacs/domdec/utility.h b/src/gromacs/domdec/utility.h
index 151cd3d833aa12034104620269612ac9a18d890c..db6ba5370138da98263755ec629454b357115bb1 100644 (file)
--- a/src/gromacs/domdec/utility.h
+++ b/src/gromacs/domdec/utility.h
@@ -54,12 +54,19 @@ static inline bool isDlbOn(const gmx_domdec_comm_t *comm)
             comm->dlbState == DlbState::onUser);
 };
 
+/*! \brief Returns true if the DLB state indicates that the balancer is off/disabled.
+ */
+static inline bool isDlbDisabled(const DlbState &dlbState)
+{
+    return (dlbState == DlbState::offUser ||
+            dlbState == DlbState::offForever);
+};
+
 /*! \brief Returns true if the DLB state indicates that the balancer is off/disabled.
  */
 static inline bool isDlbDisabled(const gmx_domdec_comm_t *comm)
 {
-    return (comm->dlbState == DlbState::offUser ||
-            comm->dlbState == DlbState::offForever);
+    return isDlbDisabled(comm->dlbState);
 };
 
 /*! \brief Returns the character, x/y/z, corresponding to dimension dim */