From 1a7f06f6a9d7b755ab3aaa7d66056786f5e6a9e0 Mon Sep 17 00:00:00 2001
From: Berk Hess <hess@kth.se>
Date: Fri, 12 Apr 2013 14:47:34 +0200
Subject: [PATCH] improved load balancing with Verlet cut-off scheme

With the Verlet scheme the force calculation order is now like group:
non-bonded, bonded, PME
The non-bonded were done after PME, which prevented load balancing,
unless separate PME nodes were used.

Change-Id: I81588ac810ce7c022bd6809f29a19c41ba13449f
---
 src/mdlib/sim_util.c | 71 ++++++++++++++++++++++++--------------------
 1 file changed, 39 insertions(+), 32 deletions(-)

diff --git a/src/mdlib/sim_util.c b/src/mdlib/sim_util.c
index 4156a0b1b1..72251d78bb 100644
--- a/src/mdlib/sim_util.c
+++ b/src/mdlib/sim_util.c
@@ -1162,42 +1162,18 @@ void do_force_cutsVERLET(FILE *fplog, t_commrec *cr,
 
         GMX_BARRIER(cr->mpi_comm_mygroup);
     }
+
     if (inputrec->ePull == epullCONSTRAINT)
     {
         clear_pull_forces(inputrec->pull);
     }
 
-    /* update QMMMrec, if necessary */
-    if (fr->bQMMM)
-    {
-        update_QMMMrec(cr, fr, x, mdatoms, box, top);
-    }
-
-    if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_POSRES].nr > 0)
-    {
-        posres_wrapper(fplog, flags, bSepDVDL, inputrec, nrnb, top, box, x,
-                       f, enerd, lambda, fr);
-    }
-
-    /* Compute the bonded and non-bonded energies and optionally forces */
-    do_force_lowlevel(fplog, step, fr, inputrec, &(top->idef),
-                      cr, nrnb, wcycle, mdatoms, &(inputrec->opts),
-                      x, hist, f, bSepLRF ? fr->f_twin : f, enerd, fcd, mtop, top, fr->born,
-                      &(top->atomtypes), bBornRadii, box,
-                      inputrec->fepvals, lambda, graph, &(top->excls), fr->mu_tot,
-                      flags, &cycles_pme);
-
-    if (bSepLRF)
-    {
-        if (do_per_step(step, inputrec->nstcalclr))
-        {
-            /* Add the long range forces to the short range forces */
-            for (i = 0; i < fr->natoms_force_constr; i++)
-            {
-                rvec_add(fr->f_twin[i], f[i], f[i]);
-            }
-        }
-    }
+    /* We calculate the non-bonded forces, when done on the CPU, here.
+     * We do this before calling do_force_lowlevel, as in there bondeds
+     * forces are calculated before PME, which does communication.
+     * With this order, non-bonded and bonded force calculation imbalance
+     * can be balanced out by the domain decomposition load balancing.
+     */
 
     if (!bUseOrEmulGPU)
     {
@@ -1206,7 +1182,6 @@ void do_force_cutsVERLET(FILE *fplog, t_commrec *cr,
                      nrnb, wcycle);
     }
 
-
     if (!bUseOrEmulGPU || bDiffKernels)
     {
         int aloc;
@@ -1248,6 +1223,38 @@ void do_force_cutsVERLET(FILE *fplog, t_commrec *cr,
         }
     }
 
+    /* update QMMMrec, if necessary */
+    if (fr->bQMMM)
+    {
+        update_QMMMrec(cr, fr, x, mdatoms, box, top);
+    }
+
+    if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_POSRES].nr > 0)
+    {
+        posres_wrapper(fplog, flags, bSepDVDL, inputrec, nrnb, top, box, x,
+                       f, enerd, lambda, fr);
+    }
+
+    /* Compute the bonded and non-bonded energies and optionally forces */
+    do_force_lowlevel(fplog, step, fr, inputrec, &(top->idef),
+                      cr, nrnb, wcycle, mdatoms, &(inputrec->opts),
+                      x, hist, f, bSepLRF ? fr->f_twin : f, enerd, fcd, mtop, top, fr->born,
+                      &(top->atomtypes), bBornRadii, box,
+                      inputrec->fepvals, lambda, graph, &(top->excls), fr->mu_tot,
+                      flags, &cycles_pme);
+
+    if (bSepLRF)
+    {
+        if (do_per_step(step, inputrec->nstcalclr))
+        {
+            /* Add the long range forces to the short range forces */
+            for (i = 0; i < fr->natoms_force_constr; i++)
+            {
+                rvec_add(fr->f_twin[i], f[i], f[i]);
+            }
+        }
+    }
+
     cycles_force += wallcycle_stop(wcycle, ewcFORCE);
     GMX_BARRIER(cr->mpi_comm_mygroup);
 
-- 
2.22.0