From: Berk Hess <hess@kth.se>
Date: Tue, 16 Jun 2015 18:16:56 +0000 (+0200)
Subject: Added missing DD cycle counting
X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=033fdfcccabee5818d49e344ebdd465ec704ef9c;p=alexxy%2Fgromacs.git

Added missing DD cycle counting

Cycle counting was missing for DD repartitioning after replica
exchange or coord swap. Removed DD cycle counting for initial DD.
Moved DD cycle counting into dd_partition_system and added
subract_cycles function with assertion to detect cycle wrapping.

Fixes #1677.

Change-Id: I7f1b19397b36456f1d120dbc0080146a384def5a
---

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index 28ffa7ab2d..0b6001e667 100644
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -9329,6 +9329,8 @@ void dd_partition_system(FILE                *fplog,
     real               grid_density;
     char               sbuf[22];
 
+    wallcycle_start(wcycle, ewcDOMDEC);
+
     dd   = cr->dd;
     comm = dd->comm;
 
@@ -9915,4 +9917,6 @@ void dd_partition_system(FILE                *fplog,
         check_index_consistency(dd, top_global->natoms, ncg_mtop(top_global),
                                 "after partitioning");
     }
+
+    wallcycle_stop(wcycle, ewcDOMDEC);
 }
diff --git a/src/gromacs/mdlib/minimize.cpp b/src/gromacs/mdlib/minimize.cpp
index b29a8dc7e2..e70ceb8be2 100644
--- a/src/gromacs/mdlib/minimize.cpp
+++ b/src/gromacs/mdlib/minimize.cpp
@@ -673,14 +673,12 @@ static void em_dd_partition_system(FILE *fplog, int step, t_commrec *cr,
                                    t_nrnb *nrnb, gmx_wallcycle_t wcycle)
 {
     /* Repartition the domain decomposition */
-    wallcycle_start(wcycle, ewcDOMDEC);
     dd_partition_system(fplog, step, cr, FALSE, 1,
                         NULL, top_global, ir,
                         &ems->s, &ems->f,
                         mdatoms, top, fr, vsite, NULL, constr,
                         nrnb, wcycle, FALSE);
     dd_store_state(cr->dd, &ems->s);
-    wallcycle_stop(wcycle, ewcDOMDEC);
 }
 
 static void evaluate_energy(FILE *fplog, t_commrec *cr,
diff --git a/src/gromacs/timing/wallcycle.c b/src/gromacs/timing/wallcycle.c
index b0b6a70852..4961ce58a3 100644
--- a/src/gromacs/timing/wallcycle.c
+++ b/src/gromacs/timing/wallcycle.c
@@ -388,6 +388,17 @@ static gmx_bool is_pme_subcounter(int ewc)
     return (ewc >= ewcPME_REDISTXF && ewc < ewcPMEWAITCOMM);
 }
 
+/* Subtract counter ewc_sub timed inside a timing block for ewc_main */
+static void subtract_cycles(wallcc_t *wcc, int ewc_main, int ewc_sub)
+{
+    if (wcc[ewc_sub].n > 0)
+    {
+        assert(wcc[ewc_main].c >= wcc[ewc_sub].c);
+
+        wcc[ewc_main].c -= wcc[ewc_sub].c;
+    }
+}
+
 void wallcycle_sum(t_commrec *cr, gmx_wallcycle_t wc)
 {
     wallcc_t *wcc;
@@ -439,26 +450,15 @@ void wallcycle_sum(t_commrec *cr, gmx_wallcycle_t wc)
         }
     }
 
-    if (wcc[ewcDDCOMMLOAD].n > 0)
-    {
-        wcc[ewcDOMDEC].c -= wcc[ewcDDCOMMLOAD].c;
-    }
-    if (wcc[ewcDDCOMMBOUND].n > 0)
-    {
-        wcc[ewcDOMDEC].c -= wcc[ewcDDCOMMBOUND].c;
-    }
-    if (wcc[ewcPME_FFTCOMM].n > 0)
-    {
-        wcc[ewcPME_FFT].c -= wcc[ewcPME_FFTCOMM].c;
-    }
+    subtract_cycles(wcc, ewcDOMDEC, ewcDDCOMMLOAD);
+    subtract_cycles(wcc, ewcDOMDEC, ewcDDCOMMBOUND);
+
+    subtract_cycles(wcc, ewcPME_FFT, ewcPME_FFTCOMM);
 
     if (cr->npmenodes == 0)
     {
         /* All nodes do PME (or no PME at all) */
-        if (wcc[ewcPMEMESH].n > 0)
-        {
-            wcc[ewcFORCE].c -= wcc[ewcPMEMESH].c;
-        }
+        subtract_cycles(wcc, ewcFORCE, ewcPMEMESH);
     }
     else
     {
@@ -466,6 +466,7 @@ void wallcycle_sum(t_commrec *cr, gmx_wallcycle_t wc)
         if (wcc[ewcPMEMESH].n > 0)
         {
             /* This must be a PME only node, calculate the Wait + Comm. time */
+            assert(wcc[ewcRUN].c >= wcc[ewcPMEMESH].c);
             wcc[ewcPMEWAITCOMM].c = wcc[ewcRUN].c - wcc[ewcPMEMESH].c;
         }
     }
diff --git a/src/programs/mdrun/md.cpp b/src/programs/mdrun/md.cpp
index 4c34f399c1..c377038774 100644
--- a/src/programs/mdrun/md.cpp
+++ b/src/programs/mdrun/md.cpp
@@ -414,7 +414,7 @@ double do_md(FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[],
                             state_global, top_global, ir,
                             state, &f, mdatoms, top, fr,
                             vsite, shellfc, constr,
-                            nrnb, wcycle, FALSE);
+                            nrnb, NULL, FALSE);
 
     }
 
@@ -917,7 +917,6 @@ double do_md(FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[],
             if (DOMAINDECOMP(cr))
             {
                 /* Repartition the domain decomposition */
-                wallcycle_start(wcycle, ewcDOMDEC);
                 dd_partition_system(fplog, step, cr,
                                     bMasterState, nstglobalcomm,
                                     state_global, top_global, ir,
@@ -925,7 +924,6 @@ double do_md(FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[],
                                     vsite, shellfc, constr,
                                     nrnb, wcycle,
                                     do_verbose && !bPMETunePrinting);
-                wallcycle_stop(wcycle, ewcDOMDEC);
             }
         }