improved PME load balancing reporting

With the group cut-off scheme, rlist was reported incorrectly.
Added a note in log and stderr when PME load balancing increases
the non-bonded workload by more than 50%.

Change-Id: I7d46f5a110b24cbb40b6b9cb53a1d90715c7c5ec

diff --git a/src/kernel/md.c b/src/kernel/md.c
index ffbcef6ea60c8a50b1a2091f83ff447105fb1774..ce7479b133237fca2f3fdab725b6f99a197d4d46 100644 (file)
--- a/src/kernel/md.c
+++ b/src/kernel/md.c
@@ -2222,7 +2222,8 @@ double do_md(FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[],
 
     if (pme_loadbal != NULL)
     {
-        pme_loadbal_done(pme_loadbal, fplog);
+        pme_loadbal_done(pme_loadbal, cr, fplog,
+                         fr->nbv != NULL && fr->nbv->bUseGPU);
     }
 
     if (shellfc && fplog)

diff --git a/src/kernel/pme_loadbal.c b/src/kernel/pme_loadbal.c
index c6c11bc13265785a609a7c2b60ed1d4345989c65..5e57d1c4e151f9cd12c3c35fad045069fd4e3c51 100644 (file)
--- a/src/kernel/pme_loadbal.c
+++ b/src/kernel/pme_loadbal.c
@@ -47,6 +47,7 @@
 #include "domdec.h"
 #include "nbnxn_cuda_data_mgmt.h"
 #include "force.h"
+#include "md_logging.h"
 #include "pme_loadbal.h"
 
 /* Parameters and setting for one PP-PME setup */
@@ -702,6 +703,22 @@ static int pme_grid_points(const pme_setup_t *setup)
     return setup->grid[XX]*setup->grid[YY]*setup->grid[ZZ];
 }
 
+static real pme_loadbal_rlist(const pme_setup_t *setup)
+{
+    /* With the group cut-off scheme we can have twin-range either
+     * for Coulomb or for VdW, so we need a check here.
+     * With the Verlet cut-off scheme rlist=rlistlong.
+     */
+    if (setup->rcut_coulomb > setup->rlist)
+    {
+        return setup->rlistlong;
+    }
+    else
+    {
+        return setup->rlist;
+    }
+}
+
 static void print_pme_loadbal_setting(FILE              *fplog,
                                       char              *name,
                                       const pme_setup_t *setup)
@@ -709,17 +726,19 @@ static void print_pme_loadbal_setting(FILE              *fplog,
     fprintf(fplog,
             "   %-7s %6.3f nm %6.3f nm     %3d %3d %3d   %5.3f nm  %5.3f nm\n",
             name,
-            setup->rcut_coulomb, setup->rlist,
+            setup->rcut_coulomb, pme_loadbal_rlist(setup),
             setup->grid[XX], setup->grid[YY], setup->grid[ZZ],
             setup->spacing, 1/setup->ewaldcoeff);
 }
 
 static void print_pme_loadbal_settings(pme_load_balancing_t pme_lb,
-                                       FILE                *fplog)
+                                       t_commrec           *cr,
+                                       FILE                *fplog,
+                                       gmx_bool             bNonBondedOnGPU)
 {
     double pp_ratio, grid_ratio;
 
-    pp_ratio   = pow(pme_lb->setup[pme_lb->cur].rlist/pme_lb->setup[0].rlistlong, 3.0);
+    pp_ratio   = pow(pme_loadbal_rlist(&pme_lb->setup[pme_lb->cur])/pme_loadbal_rlist(&pme_lb->setup[0]), 3.0);
     grid_ratio = pme_grid_points(&pme_lb->setup[pme_lb->cur])/
         (double)pme_grid_points(&pme_lb->setup[0]);
 
@@ -747,14 +766,27 @@ static void print_pme_loadbal_settings(pme_load_balancing_t pme_lb,
     fprintf(fplog, " cost-ratio           %4.2f             %4.2f\n",
             pp_ratio, grid_ratio);
     fprintf(fplog, " (note that these numbers concern only part of the total PP and PME load)\n");
-    fprintf(fplog, "\n");
+
+    if (pp_ratio > 1.5 && !bNonBondedOnGPU)
+    {
+        md_print_warn(cr, fplog,
+                      "NOTE: PME load balancing increased the non-bonded workload by more than 50%%.\n"
+                      "      For better performance use (more) PME nodes (mdrun -npme),\n"
+                      "      or in case you are beyond the scaling limit, use less nodes in total.\n");
+    }
+    else
+    {
+        fprintf(fplog, "\n");
+    }
 }
 
-void pme_loadbal_done(pme_load_balancing_t pme_lb, FILE *fplog)
+void pme_loadbal_done(pme_load_balancing_t pme_lb,
+                      t_commrec *cr, FILE *fplog,
+                      gmx_bool bNonBondedOnGPU)
 {
     if (fplog != NULL && (pme_lb->cur > 0 || pme_lb->elimited != epmelblimNO))
     {
-        print_pme_loadbal_settings(pme_lb, fplog);
+        print_pme_loadbal_settings(pme_lb, cr, fplog, bNonBondedOnGPU);
     }
 
     /* TODO: Here we should free all pointers in pme_lb,

diff --git a/src/kernel/pme_loadbal.h b/src/kernel/pme_loadbal.h
index a444fae14c361e6f54d8a4ecdbc8acee794beb48..99016d0163f535381c09da00aa0f2064991806e7 100644 (file)
--- a/src/kernel/pme_loadbal.h
+++ b/src/kernel/pme_loadbal.h
@@ -75,6 +75,8 @@ gmx_bool pme_load_balance(pme_load_balancing_t pme_lb,
 void restart_pme_loadbal(pme_load_balancing_t pme_lb, int n);
 
 /* Finish the PME load balancing and print the settings when fplog!=NULL */
-void pme_loadbal_done(pme_load_balancing_t pme_lb, FILE *fplog);
+void pme_loadbal_done(pme_load_balancing_t pme_lb,
+                      t_commrec *cr, FILE *fplog,
+                      gmx_bool bNonBondedOnGPU);
 
 #endif /* _pme_loadbal_h */