Added functionality that allows MdModules to disable PME-only ranks during simulation...

[alexxy/gromacs.git] / src / gromacs / mdrun / runner.cpp

diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp
index 943913668c7dbe98a36ea5220df26080b047c622..8ce13dfff4d52f3934b0041b9545605b740c2053 100644 (file)
--- a/src/gromacs/mdrun/runner.cpp
+++ b/src/gromacs/mdrun/runner.cpp
@@ -64,6 +64,7 @@
 #include "gromacs/domdec/localatomsetmanager.h"
 #include "gromacs/domdec/partition.h"
 #include "gromacs/ewald/ewald_utils.h"
+#include "gromacs/ewald/pme.h"
 #include "gromacs/ewald/pme_gpu_program.h"
 #include "gromacs/ewald/pme_only.h"
 #include "gromacs/ewald/pme_pp_comm_gpu.h"
@@ -1061,40 +1062,59 @@ int Mdrunner::mdrunner()
                   "these are not compatible with mdrun -rerun");
     }
 
+    /* Object for collecting reasons for not using PME-only ranks */
+    SeparatePmeRanksPermitted separatePmeRanksPermitted;
+
+    /* Permit MDModules to notify whether they want to use PME-only ranks */
+    mdModulesNotifier.notify(&separatePmeRanksPermitted);
+
+    /* If simulation is not using PME then disable PME-only ranks */
     if (!(EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)))
     {
-        if (domdecOptions.numPmeRanks > 0)
-        {
-            gmx_fatal_collective(FARGS,
-                                 cr->mpiDefaultCommunicator,
-                                 MASTER(cr),
-                                 "PME-only ranks are requested, but the system does not use PME "
-                                 "for electrostatics or LJ");
-        }
-
-        domdecOptions.numPmeRanks = 0;
+        separatePmeRanksPermitted.disablePmeRanks(
+                "PME-only ranks are requested, but the system does not use PME "
+                "for electrostatics or LJ");
     }
 
+    /* With NB GPUs we don't automatically use PME-only CPU ranks. PME ranks can
+     * improve performance with many threads per GPU, since our OpenMP
+     * scaling is bad, but it's difficult to automate the setup.
+     */
     if (useGpuForNonbonded && domdecOptions.numPmeRanks < 0)
     {
-        /* With NB GPUs we don't automatically use PME-only CPU ranks. PME ranks can
-         * improve performance with many threads per GPU, since our OpenMP
-         * scaling is bad, but it's difficult to automate the setup.
-         */
+        separatePmeRanksPermitted.disablePmeRanks(
+                "PME-only CPU ranks are not automatically used when "
+                "non-bonded interactions are computed on GPUs");
+    }
+
+    /* If GPU is used for PME then only 1 PME rank is permitted */
+    if (useGpuForPme && (domdecOptions.numPmeRanks < 0 || domdecOptions.numPmeRanks > 1))
+    {
+        separatePmeRanksPermitted.disablePmeRanks(
+                "PME GPU decomposition is not supported. Only one separate PME-only GPU rank "
+                "can be used.");
+    }
+
+    /* Disable PME-only ranks if some parts of the code requested so and it's up to GROMACS to decide */
+    if (!separatePmeRanksPermitted.permitSeparatePmeRanks() && domdecOptions.numPmeRanks < 0)
+    {
         domdecOptions.numPmeRanks = 0;
+        GMX_LOG(mdlog.info)
+                .asParagraph()
+                .appendText("Simulation will not use PME-only ranks because: "
+                            + separatePmeRanksPermitted.reasonsWhyDisabled());
     }
-    if (useGpuForPme)
+
+    /* If some parts of the code could not use PME-only ranks and
+     * user explicitly used mdrun -npme option then throw an error */
+    if (!separatePmeRanksPermitted.permitSeparatePmeRanks() && domdecOptions.numPmeRanks > 0)
     {
-        if (domdecOptions.numPmeRanks < 0)
-        {
-            domdecOptions.numPmeRanks = 0;
-            // TODO possibly print a note that one can opt-in for a separate PME GPU rank?
-        }
-        else
-        {
-            GMX_RELEASE_ASSERT(domdecOptions.numPmeRanks <= 1,
-                               "PME GPU decomposition is not supported");
-        }
+        gmx_fatal_collective(FARGS,
+                             cr->mpiDefaultCommunicator,
+                             MASTER(cr),
+                             "Requested -npme %d option is not viable because: %s",
+                             domdecOptions.numPmeRanks,
+                             separatePmeRanksPermitted.reasonsWhyDisabled().c_str());
     }
 
     /* NMR restraints must be initialized before load_checkpoint,
@@ -1458,10 +1478,10 @@ int Mdrunner::mdrunner()
                           hw_opt.nthreads_omp_pme,
                           !thisRankHasDuty(cr, DUTY_PP));
 
-    // Enable FP exception detection, but not in
-    // Release mode and not for compilers with known buggy FP
-    // exception support (clang with any optimization) or suspected
-    // buggy FP exception support (gcc 7.* with optimization).
+// Enable FP exception detection, but not in
+// Release mode and not for compilers with known buggy FP
+// exception support (clang with any optimization) or suspected
+// buggy FP exception support (gcc 7.* with optimization).
 #if !defined NDEBUG                                                                         \
         && !((defined __clang__ || (defined(__GNUC__) && !defined(__ICC) && __GNUC__ == 7)) \
              && defined __OPTIMIZE__)