Add havePPDomainDecomposition()

When using 1 PP and 1 PME rank, non-local code paths were called
for with GPUs. Now the DOMAINDECOMP(cr) conditional for the paths
has been replaced by havePPDomainDecomposition().

Change-Id: I1b7036b0759a316c68d5c00bd5814b067c9fd9cd

diff --git a/src/gromacs/domdec/dlbtiming.h b/src/gromacs/domdec/dlbtiming.h
index 67e909f245ec0d10f07ea85bbe36cdbe78626111..8834e3d3b0dcb6b3d57b215671477c40814b1488 100644 (file)
--- a/src/gromacs/domdec/dlbtiming.h
+++ b/src/gromacs/domdec/dlbtiming.h
@@ -86,11 +86,9 @@ class DDBalanceRegionHandler
     public:
         //! Constructor, pass a pointer to t_commrec or nullptr when not using domain decomposition
         DDBalanceRegionHandler(const t_commrec *cr) :
+            useBalancingRegion_(havePPDomainDecomposition(cr)),
             dd_(cr != nullptr ? cr->dd : nullptr)
         {
-            useBalancingRegion_ = (cr != nullptr &&
-                                   cr->dd != nullptr &&
-                                   cr->nnodes - cr->npmenodes > 1);
         }
 
         /*! \brief Returns whether were are actually using the balancing region

diff --git a/src/gromacs/mdlib/sim_util.cpp b/src/gromacs/mdlib/sim_util.cpp
index eeacc953451d9b75f46e9e0b86536151f9a2ff7d..c057604c94f4927def8c7c6a4212383fc8b3e0b1 100644 (file)
--- a/src/gromacs/mdlib/sim_util.cpp
+++ b/src/gromacs/mdlib/sim_util.cpp
@@ -911,7 +911,7 @@ static inline void launchGpuRollingPruning(const t_commrec          *cr,
     bool stepIsEven          = ((stepWithCurrentList & 1) == 0);
     if (stepWithCurrentList > 0 &&
         stepWithCurrentList < inputrec->nstlist - 1 &&
-        (stepIsEven || DOMAINDECOMP(cr)))
+        (stepIsEven || havePPDomainDecomposition(cr)))
     {
         Nbnxm::gpu_launch_kernel_pruneonly(nbv->gpu_nbv,
                                            stepIsEven ? Nbnxm::InteractionLocality::Local : Nbnxm::InteractionLocality::NonLocal,
@@ -1189,7 +1189,7 @@ static void do_force_cutsVERLET(FILE *fplog,
 
         // bonded work not split into separate local and non-local, so with DD
         // we can only launch the kernel after non-local coordinates have been received.
-        if (ppForceWorkload->haveGpuBondedWork && !DOMAINDECOMP(cr))
+        if (ppForceWorkload->haveGpuBondedWork && !havePPDomainDecomposition(cr))
         {
             wallcycle_sub_start(wcycle, ewcsLAUNCH_GPU_BONDED);
             fr->gpuBonded->launchKernels(fr, flags, box);
@@ -1215,7 +1215,7 @@ static void do_force_cutsVERLET(FILE *fplog,
 
     /* Communicate coordinates and sum dipole if necessary +
        do non-local pair search */
-    if (DOMAINDECOMP(cr))
+    if (havePPDomainDecomposition(cr))
     {
         nbnxn_pairlist_set_t &pairlistSet = nbv->grp[Nbnxm::InteractionLocality::NonLocal].nbl_lists;
 
@@ -1287,7 +1287,7 @@ static void do_force_cutsVERLET(FILE *fplog,
         /* launch D2H copy-back F */
         wallcycle_start_nocount(wcycle, ewcLAUNCH_GPU);
         wallcycle_sub_start_nocount(wcycle, ewcsLAUNCH_GPU_NONBONDED);
-        if (DOMAINDECOMP(cr))
+        if (havePPDomainDecomposition(cr))
         {
             Nbnxm::gpu_launch_cpyback(nbv->gpu_nbv, nbv->nbat,
                                       flags, Nbnxm::AtomLocality::NonLocal, ppForceWorkload->haveGpuBondedWork);
@@ -1430,7 +1430,7 @@ static void do_force_cutsVERLET(FILE *fplog,
 
     if (!bUseOrEmulGPU)
     {
-        if (DOMAINDECOMP(cr))
+        if (havePPDomainDecomposition(cr))
         {
             do_nb_verlet(fr, ic, enerd, flags, Nbnxm::InteractionLocality::NonLocal, enbvClearFNo,
                          step, nrnb, wcycle);
@@ -1485,7 +1485,7 @@ static void do_force_cutsVERLET(FILE *fplog,
     if (bUseOrEmulGPU)
     {
         /* wait for non-local forces (or calculate in emulation mode) */
-        if (DOMAINDECOMP(cr))
+        if (havePPDomainDecomposition(cr))
         {
             if (bUseGPU)
             {
@@ -1514,7 +1514,7 @@ static void do_force_cutsVERLET(FILE *fplog,
         }
     }
 
-    if (DOMAINDECOMP(cr))
+    if (havePPDomainDecomposition(cr))
     {
         /* We are done with the CPU compute.
          * We will now communicate the non-local forces.

diff --git a/src/gromacs/mdtypes/commrec.h b/src/gromacs/mdtypes/commrec.h
index d9617c0012069665db8d990629d27d4f4d406787..e85805a26ac99e7f19d34d0606a0d2152010017b 100644 (file)
--- a/src/gromacs/mdtypes/commrec.h
+++ b/src/gromacs/mdtypes/commrec.h
@@ -3,7 +3,7 @@
  *
  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
  * Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2017,2018, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2017,2018,2019, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -169,7 +169,21 @@ inline bool thisRankHasDuty(const t_commrec *cr, int duty)
  * PAR(cr) and DOMAINDECOMP(cr) are not universally synonymous. In
  * particular, DOMAINDECOMP(cr) == true indicates that there is more
  * than one domain, not just that the dd algorithm is active. */
-#define DOMAINDECOMP(cr)   (((cr)->dd != NULL) && PAR(cr))
+#define DOMAINDECOMP(cr)   (((cr)->dd != nullptr) && PAR(cr))
+
+/*! \brief Returns whether we have actual domain decomposition for the particle-particle interactions
+ *
+ * Will return false when we use 1 rank for PP and 1 for PME
+ */
+static bool inline havePPDomainDecomposition(const t_commrec *cr)
+{
+    /* NOTE: It would be better to use cr->dd->nnodes, but we do not want
+     *       to pull in a dependency on domdec.h into this file.
+     */
+    return (cr != nullptr &&
+            cr->dd != nullptr &&
+            cr->nnodes - cr->npmenodes > 1);
+}
 
 //! Are we doing multiple independent simulations?
 static bool inline isMultiSim(const gmx_multisim_t *ms)