PP-PME load balancing improvements

Add a minimum number of nstlist tuning intervals and minimum time delay
at the beginning of the run before the load balancing starts. This allow
hardware clocks to ramp up and avoids having early measurements
overestimate rendering subsequent ones with different grid setups only
faster due to hardware warmup.
Also use global variables to adjust the number of measurements to be
skipped after switching configs.

Refs  #3208
Fixes #2203

Change-Id: If835d2482e127caa51d50f45f25c19144d35efaa

diff --git a/docs/release-notes/2020/major/performance.rst b/docs/release-notes/2020/major/performance.rst
index 656c9be7f16943738c9a95a5c894fe9dbe4455ae..604cd840346386a4f8d181901207f55c95a8ba3a 100644 (file)
--- a/docs/release-notes/2020/major/performance.rst
+++ b/docs/release-notes/2020/major/performance.rst
@@ -44,3 +44,10 @@ Bonded kernels on GPU have been fused
 Instead of launching one GPU kernel for each listed interaction type there is now one
 GPU kernel that handles all listed interactions. This improves the performance when
 running bonded calculations on a GPU.
+
+Delay for ramp-up added to PP-PME tuning
+""""""""""""""""""""""""""""""""""""""""
+
+Modern CPUs and GPUs can take a few seconds to ramp up their clock speeds.
+Therefore the PP-PME load balancing now starts after 5 seconds instead
+of after a few MD steps. This avoids sub-optimal performance settings.

diff --git a/src/gromacs/ewald/pme_load_balancing.cpp b/src/gromacs/ewald/pme_load_balancing.cpp
index 98d0d9e2fcc1a616f36d63ef8e72f3a51ff7f95b..5e8893e254ab5841ebca55a7741229e8e3dfe323 100644 (file)
--- a/src/gromacs/ewald/pme_load_balancing.cpp
+++ b/src/gromacs/ewald/pme_load_balancing.cpp
@@ -124,6 +124,18 @@ const real maxRelativeSlowdownAccepted = 1.12;
  */
 const real maxFluctuationAccepted = 1.02;
 
+//! \brief Number of nstlist long tuning intervals to skip before starting
+//         load-balancing at the beginning of the run.
+const int c_numFirstTuningIntervalSkip = 5;
+//! \brief Number of nstlist long tuning intervals to skip before starting
+//         load-balancing at the beginning of the run with separate PME ranks. */
+const int c_numFirstTuningIntervalSkipWithSepPme = 3;
+//! \brief Number of nstlist long tuning intervals to skip after switching to a new setting
+//         during balancing.
+const int c_numPostSwitchTuningIntervalSkip = 1;
+//! \brief Number of seconds to delay the tuning at startup to allow processors clocks to ramp up.
+const double c_startupTimeDelay = 5.0;
+
 /*! \brief Enumeration whose values describe the effect limiting the load balancing */
 enum epmelb
 {
@@ -168,8 +180,9 @@ struct pme_load_balancing_t
 
     int stage; /**< the current stage */
 
-    int    cycles_n; /**< step cycle counter cummulative count */
-    double cycles_c; /**< step cycle counter cummulative cycles */
+    int    cycles_n; /**< step cycle counter cumulative count */
+    double cycles_c; /**< step cycle counter cumulative cycles */
+    double startTime; /**< time stamp when the balancing was started (relative to the UNIX epoch start).*/
 };
 
 /* TODO The code in this file should call this getter, rather than
@@ -188,8 +201,7 @@ void pme_loadbal_init(pme_load_balancing_t**     pme_lb_p,
                       const interaction_const_t& ic,
                       const nonbonded_verlet_t&  nbv,
                       gmx_pme_t*                 pmedata,
-                      gmx_bool                   bUseGPU,
-                      gmx_bool*                  bPrinting)
+                      gmx_bool                   bUseGPU)
 {
 
     pme_load_balancing_t* pme_lb;
@@ -277,8 +289,9 @@ void pme_loadbal_init(pme_load_balancing_t**     pme_lb_p,
     pme_lb->end         = 0;
     pme_lb->elimited    = epmelblimNO;
 
-    pme_lb->cycles_n = 0;
-    pme_lb->cycles_c = 0;
+    pme_lb->cycles_n  = 0;
+    pme_lb->cycles_c  = 0;
+    pme_lb->startTime = gmx_gettime();
 
     if (!wallcycle_have_counter())
     {
@@ -322,8 +335,6 @@ void pme_loadbal_init(pme_load_balancing_t**     pme_lb_p,
     }
 
     *pme_lb_p = pme_lb;
-
-    *bPrinting = pme_lb->bBalance;
 }
 
 /*! \brief Try to increase the cutoff during load balancing */
@@ -559,18 +570,19 @@ static void pme_load_balance(pme_load_balancing_t*          pme_lb,
     set = &pme_lb->setup[pme_lb->cur];
     set->count++;
 
-    if (set->count % 2 == 1)
+    /* Skip the first c_numPostSwitchTuningIntervalSkip cycles because the first step
+     * after a switch is much slower due to allocation and/or caching effects.
+     */
+    if (set->count % (c_numPostSwitchTuningIntervalSkip + 1) != 0)
     {
-        /* Skip the first cycle, because the first step after a switch
-         * is much slower due to allocation and/or caching effects.
-         */
         return;
     }
 
     sprintf(buf, "step %4s: ", gmx_step_str(step, sbuf));
     print_grid(fp_err, fp_log, buf, "timed with", set, cycles);
 
-    if (set->count <= 2)
+    GMX_RELEASE_ASSERT(set->count > c_numPostSwitchTuningIntervalSkip, "We should skip cycles");
+    if (set->count == (c_numPostSwitchTuningIntervalSkip + 1))
     {
         set->cycles = cycles;
     }
@@ -905,9 +917,14 @@ void pme_loadbal_do(pme_load_balancing_t*          pme_lb,
 
     /* Before the first step we haven't done any steps yet.
      * Also handle cases where ir.init_step % ir.nstlist != 0.
+     * We also want to skip a number of steps and seconds while
+     * the CPU and GPU, when used, performance stabilizes.
      */
-    if (pme_lb->cycles_n < ir.nstlist)
+    if (pme_lb->cycles_n == 0 || step_rel < c_numFirstTuningIntervalSkip * ir.nstlist
+        || gmx_gettime() - pme_lb->startTime < c_startupTimeDelay)
     {
+        *bPrinting = FALSE;
+
         return;
     }
     /* Sanity check, we expect nstlist cycle counts */
@@ -930,7 +947,7 @@ void pme_loadbal_do(pme_load_balancing_t*          pme_lb,
          * is not over the last nstlist steps, but the nstlist steps before
          * that. So the first useful ratio is available at step_rel=3*nstlist.
          */
-        else if (step_rel >= 3 * ir.nstlist)
+        else if (step_rel >= c_numFirstTuningIntervalSkipWithSepPme * ir.nstlist)
         {
             if (DDMASTER(cr->dd))
             {

diff --git a/src/gromacs/ewald/pme_load_balancing.h b/src/gromacs/ewald/pme_load_balancing.h
index 929188917aed6f6ffa0668f586d52da02cbc7eb3..028ebcebd9ceb0631f40ca4789badfca181ce020 100644 (file)
--- a/src/gromacs/ewald/pme_load_balancing.h
+++ b/src/gromacs/ewald/pme_load_balancing.h
@@ -72,7 +72,6 @@ bool pme_loadbal_is_active(const pme_load_balancing_t* pme_lb);
  *
  * Initialize the PP-PME load balacing data and infrastructure.
  * The actual load balancing might start right away, later or never.
- * Returns in bPrinting whether the load balancing is printing to fp_err.
  * The PME grid in pmedata is reused for smaller grids to lower the memory
  * usage.
  */
@@ -84,8 +83,7 @@ void pme_loadbal_init(pme_load_balancing_t**     pme_lb_p,
                       const interaction_const_t& ic,
                       const nonbonded_verlet_t&  nbv,
                       gmx_pme_t*                 pmedata,
-                      gmx_bool                   bUseGPU,
-                      gmx_bool*                  bPrinting);
+                      gmx_bool                   bUseGPU);
 
 /*! \brief Process cycles and PME load balance when necessary
  *

diff --git a/src/gromacs/mdrun/md.cpp b/src/gromacs/mdrun/md.cpp
index e8faae290acede997aff578c0704ce93af9432db..77b80ee5b99285251abcb6c423076a95978f5326 100644 (file)
--- a/src/gromacs/mdrun/md.cpp
+++ b/src/gromacs/mdrun/md.cpp
@@ -433,7 +433,7 @@ void gmx::LegacySimulator::do_md()
     if (bPMETune)
     {
         pme_loadbal_init(&pme_loadbal, cr, mdlog, *ir, state->box, *fr->ic, *fr->nbv, fr->pmedata,
-                         fr->nbv->useGpu(), &bPMETunePrinting);
+                         fr->nbv->useGpu());
     }
 
     if (!ir->bContinuation)

diff --git a/src/gromacs/modularsimulator/pmeloadbalancehelper.cpp b/src/gromacs/modularsimulator/pmeloadbalancehelper.cpp
index 718fc2ccbf1155e2f14f2acf6b1942190c3c28f7..bd5d9a94b5c0993f1a33f09d694915ad9b30a64c 100644 (file)
--- a/src/gromacs/modularsimulator/pmeloadbalancehelper.cpp
+++ b/src/gromacs/modularsimulator/pmeloadbalancehelper.cpp
@@ -91,7 +91,7 @@ void PmeLoadBalanceHelper::setup()
     GMX_RELEASE_ASSERT(box[0][0] != 0 && box[1][1] != 0 && box[2][2] != 0,
                        "PmeLoadBalanceHelper cannot be initialized with zero box.");
     pme_loadbal_init(&pme_loadbal_, cr_, mdlog_, *inputrec_, box, *fr_->ic, *fr_->nbv, fr_->pmedata,
-                     fr_->nbv->useGpu(), &bPMETunePrinting_);
+                     fr_->nbv->useGpu());
 }
 
 void PmeLoadBalanceHelper::run(gmx::Step step, gmx::Time gmx_unused time)