Clean up options for domain decomposition

Documented mdrun options better. Documented the environment variables.
Renamed the environment variables to better describe their function.
Renamed the helper function because not all of these environment
variables relate to numbers of steps.

Change-Id: I12dba40f8cdf489ea23c3f4dd8810efde54a4aaf

diff --git a/manual/install.tex b/manual/install.tex
index 91d62cd3e518c299ead99c711e4ef3e53b23e939..825e9dd8c5dbe6df138f17f63f3ec4cbe77344ee 100644 (file)
--- a/manual/install.tex
+++ b/manual/install.tex
@@ -209,8 +209,20 @@ you should consult your local documentation for details.
         Mainly for debugging purposes, but may also be handy for
         porting to other platforms.
 \item   {\tt WHERE}: when set, print debugging info on line numbers.
+\item   {\tt GMX_DD_NST_DUMP}: number of steps that elapse between dumping
+        the current DD to a PDB file (default 0). This only takes effect
+        during domain decomposition, so it should typically be
+        0 (never), 1 (every DD phase) or a multiple of {\tt nstlist}.
+\item   {\tt GMX_DD_NST_DUMP_GRID}: number of steps that elapse between dumping
+        the current DD grid to a PDB file (default 0). This only takes effect
+        during domain decomposition, so it should typically be
+        0 (never), 1 (every DD phase) or a multiple of {\tt nstlist}.
+\item   {\tt GMX_DD_DEBUG}: general debugging trigger for every domain
+        decomposition (default 0, meaning off). Currently only checks
+        global-local atom index mapping for consistency.
+\item   {\tt GMX_DD_NPULSE}: over-ride the number of DD pulses used
+        (default 0, meaning no over-ride). Normally 1 or 2.
 
-% At this point, all environment variables should be described
 %\item   There are a number of extra environment variables like these
 %        that are used in debugging - check the code!
 
@@ -241,6 +253,19 @@ you should consult your local documentation for details.
 \item   {\tt GMX_CYCLE_BARRIER}: calls MPI_Barrier before each cycle start/stop call.
 \item   {\tt GMX_DD_ORDER_ZYX}: build domain decomposition cells in the order
         (z, y, x) rather than the default (x, y, z).
+\item   {\tt GMX_DD_USE_SENDRECV2}: during constraint and vsite communication, use a pair
+        of {\tt MPI_SendRecv} calls instead of two simultaneous non-blocking calls
+        (default 0, meaning off). Might be faster on some MPI implementations.
+\item   {\tt GMX_DLB_BASED_ON_FLOPS}: do domain-decomposition dynamic load balancing based on flop count rather than
+        measured time elapsed (default 0, meaning off).
+        This makes the load balancing reproducible, which can be useful for debugging purposes.
+        A value of 1 uses the flops; a value > 1 adds (value - 1)*5\% of noise to the flops to increase the imbalance and the scaling.
+\item   {\tt GMX_DLB_MAX_BOX_SCALING}: maximum percentage box scaling permitted per domain-decomposition
+        load-balancing step (default 10)
+\item   {\tt GMX_DD_RECORD_LOAD}: record DD load statistics for reporting at end of the run (default 1, meaning on)
+\item   {\tt GMX_DD_NST_SORT_CHARGE_GROUPS}: number of steps that elapse between re-sorting of the charge
+        groups (default 1). This only takes effect during domain decomposition, so should typically
+        be 0 (never), 1 (to mean at every domain decomposition), or a multiple of {\tt nstlist}.
 \item   {\tt GMX_DETAILED_PERF_STATS}: when set, print slightly more detailed performance information
         to the {\tt .log} file. The resulting output is the way performance summary is reported in versions
         4.5.x and thus may be useful for anyone using scripts to parse {\tt .log} files or standard output.

diff --git a/src/gromacs/mdlib/domdec.c b/src/gromacs/mdlib/domdec.c
index 0d5a51049a7354dfe3db6f9eb3662235cd9c9caa..864c05435879abcc444f616c4a267f0d2c81a90e 100644 (file)
--- a/src/gromacs/mdlib/domdec.c
+++ b/src/gromacs/mdlib/domdec.c
@@ -6423,7 +6423,7 @@ static int multi_body_bondeds_count(gmx_mtop_t *mtop)
     return n;
 }
 
-static int dd_nst_env(FILE *fplog, const char *env_var, int def)
+static int dd_getenv(FILE *fplog, const char *env_var, int def)
 {
     char *val;
     int   nst;
@@ -6666,14 +6666,14 @@ gmx_domdec_t *init_domain_decomposition(FILE *fplog, t_commrec *cr,
     dd->npbcdim   = ePBC2npbcdim(ir->ePBC);
     dd->bScrewPBC = (ir->ePBC == epbcSCREW);
 
-    dd->bSendRecv2      = dd_nst_env(fplog, "GMX_DD_SENDRECV2", 0);
-    comm->dlb_scale_lim = dd_nst_env(fplog, "GMX_DLB_MAX", 10);
-    comm->eFlop         = dd_nst_env(fplog, "GMX_DLB_FLOP", 0);
-    recload             = dd_nst_env(fplog, "GMX_DD_LOAD", 1);
-    comm->nstSortCG     = dd_nst_env(fplog, "GMX_DD_SORT", 1);
-    comm->nstDDDump     = dd_nst_env(fplog, "GMX_DD_DUMP", 0);
-    comm->nstDDDumpGrid = dd_nst_env(fplog, "GMX_DD_DUMP_GRID", 0);
-    comm->DD_debug      = dd_nst_env(fplog, "GMX_DD_DEBUG", 0);
+    dd->bSendRecv2      = dd_getenv(fplog, "GMX_DD_USE_SENDRECV2", 0);
+    comm->dlb_scale_lim = dd_getenv(fplog, "GMX_DLB_MAX_BOX_SCALING", 10);
+    comm->eFlop         = dd_getenv(fplog, "GMX_DLB_BASED_ON_FLOPS", 0);
+    recload             = dd_getenv(fplog, "GMX_DD_RECORD_LOAD", 1);
+    comm->nstSortCG     = dd_getenv(fplog, "GMX_DD_NST_SORT_CHARGE_GROUPS", 1);
+    comm->nstDDDump     = dd_getenv(fplog, "GMX_DD_NST_DUMP", 0);
+    comm->nstDDDumpGrid = dd_getenv(fplog, "GMX_DD_NST_DUMP_GRID", 0);
+    comm->DD_debug      = dd_getenv(fplog, "GMX_DD_DEBUG", 0);
 
     dd->pme_recv_f_alloc = 0;
     dd->pme_recv_f_buf   = NULL;
@@ -7361,7 +7361,7 @@ static void set_cell_limits_dlb(gmx_domdec_t      *dd,
     }
 
     /* This env var can override npulse */
-    d = dd_nst_env(debug, "GMX_DD_NPULSE", 0);
+    d = dd_getenv(debug, "GMX_DD_NPULSE", 0);
     if (d > 0)
     {
         npulse = d;

diff --git a/src/programs/mdrun/mdrun.cpp b/src/programs/mdrun/mdrun.cpp
index 0c7ece1adf3c1e81c852ae2161ccd83aef93f34c..332d64e5e83cafaeba1b9d77ee7d430bdfa08dcf 100644 (file)
--- a/src/programs/mdrun/mdrun.cpp
+++ b/src/programs/mdrun/mdrun.cpp
@@ -507,13 +507,20 @@ int gmx_mdrun(int argc, char *argv[])
         { "-dlb",     FALSE, etENUM, {dddlb_opt},
           "Dynamic load balancing (with DD)" },
         { "-dds",     FALSE, etREAL, {&dlb_scale},
-          "Minimum allowed dlb scaling of the DD cell size" },
+          "Fraction in (0,1) by whose reciprocal the initial DD cell size will be increased in order to "
+          "provide a margin in which dynamic load balancing can act while preserving the minimum cell size." },
         { "-ddcsx",   FALSE, etSTR, {&ddcsx},
-          "HIDDENThe DD cell sizes in x" },
+          "HIDDENA string containing a vector of the relative sizes in the x "
+          "direction of the corresponding DD cells. Only effective with static "
+          "load balancing." },
         { "-ddcsy",   FALSE, etSTR, {&ddcsy},
-          "HIDDENThe DD cell sizes in y" },
+          "HIDDENA string containing a vector of the relative sizes in the y "
+          "direction of the corresponding DD cells. Only effective with static "
+          "load balancing." },
         { "-ddcsz",   FALSE, etSTR, {&ddcsz},
-          "HIDDENThe DD cell sizes in z" },
+          "HIDDENA string containing a vector of the relative sizes in the z "
+          "direction of the corresponding DD cells. Only effective with static "
+          "load balancing." },
         { "-gcom",    FALSE, etINT, {&nstglobalcomm},
           "Global communication frequency" },
         { "-nb",      FALSE, etENUM, {&nbpu_opt},