Merge release-2018 into master

[alexxy/gromacs.git] / src / gromacs / taskassignment / resourcedivision.cpp

diff --git a/src/gromacs/taskassignment/resourcedivision.cpp b/src/gromacs/taskassignment/resourcedivision.cpp
index 115dcb650f0541f7e0ae2d6abb52e6a176d68d1c..309d054f494f14e57332fe0316a116f0af366f73 100644 (file)
--- a/src/gromacs/taskassignment/resourcedivision.cpp
+++ b/src/gromacs/taskassignment/resourcedivision.cpp
@@ -67,6 +67,7 @@
 #include "gromacs/utility/fatalerror.h"
 #include "gromacs/utility/gmxassert.h"
 #include "gromacs/utility/logger.h"
+#include "gromacs/utility/physicalnodecommunicator.h"
 #include "gromacs/utility/stringutil.h"
 
 
@@ -303,7 +304,7 @@ class SingleRankChecker
 {
     public:
         //! Constructor
-        SingleRankChecker() : value_(false), reasons_() {}
+        SingleRankChecker() : value_(false) {}
         /*! \brief Call this function for each possible condition
             under which a single rank is required, along with a string
             describing the constraint when it is applied. */
@@ -360,7 +361,7 @@ int get_nthreads_mpi(const gmx_hw_info_t    *hwinfo,
     if (pmeOnGpu)
     {
         GMX_RELEASE_ASSERT((EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)) &&
-                           pme_gpu_supports_input(inputrec, nullptr),
+                           pme_gpu_supports_build(nullptr) && pme_gpu_supports_input(inputrec, nullptr),
                            "PME can't be on GPUs unless we are using PME");
 
         // PME on GPUs supports a single PME rank with PP running on the same or few other ranks.
@@ -794,11 +795,13 @@ void check_and_update_hw_opt_2(gmx_hw_opt_t *hw_opt,
     }
 }
 
-void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t        *hw_opt,
-                                             const gmx_hw_info_t &hwinfo,
-                                             const t_commrec     *cr,
-                                             PmeRunMode           pmeRunMode,
-                                             const gmx_mtop_t    &mtop)
+void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t         *hw_opt,
+                                             const gmx_hw_info_t  &hwinfo,
+                                             const t_commrec      *cr,
+                                             const gmx_multisim_t *ms,
+                                             int                   numRanksOnThisNode,
+                                             PmeRunMode            pmeRunMode,
+                                             const gmx_mtop_t     &mtop)
 {
 #if GMX_THREAD_MPI
     GMX_RELEASE_ASSERT(hw_opt->nthreads_tmpi >= 1, "Must have at least one thread-MPI rank");
@@ -851,14 +854,13 @@ void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t        *hw_opt,
          * all detected ncore_tot physical cores. We are currently not
          * checking for that here.
          */
-        int numRanksTot     = cr->nnodes*(MULTISIM(cr) ? cr->ms->nsim : 1);
+        int numRanksTot     = cr->nnodes*(isMultiSim(ms) ? ms->nsim : 1);
         int numAtomsPerRank = mtop.natoms/cr->nnodes;
         int numCoresPerRank = hwinfo.ncore_tot/numRanksTot;
         if (numAtomsPerRank < c_numAtomsPerCoreSquaredSmtThreshold*gmx::square(numCoresPerRank))
         {
-            int numRanksInThisNode = (cr ? cr->nrank_intranode : 1);
             /* Choose one OpenMP thread per physical core */
-            hw_opt->nthreads_omp = std::max(1, hwinfo.hardwareTopology->numberOfCores()/numRanksInThisNode);
+            hw_opt->nthreads_omp = std::max(1, hwinfo.hardwareTopology->numberOfCores()/numRanksOnThisNode);
         }
     }
 
@@ -876,45 +878,49 @@ void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t        *hw_opt,
     }
 }
 
-void checkHardwareOversubscription(int                          numThreadsOnThisRank,
-                                   const gmx::HardwareTopology &hwTop,
-                                   const t_commrec             *cr,
-                                   const gmx::MDLogger         &mdlog)
+namespace gmx
 {
-    if (hwTop.supportLevel() < gmx::HardwareTopology::SupportLevel::LogicalProcessorCount)
+
+void checkHardwareOversubscription(int                             numThreadsOnThisRank,
+                                   int                             rank,
+                                   const HardwareTopology         &hwTop,
+                                   const PhysicalNodeCommunicator &comm,
+                                   const MDLogger                 &mdlog)
+{
+    if (hwTop.supportLevel() < HardwareTopology::SupportLevel::LogicalProcessorCount)
     {
         /* There is nothing we can check */
         return;
     }
 
-    int numRanksOnThisNode   = 1;
+    int numRanksOnThisNode   = comm.size_;
     int numThreadsOnThisNode = numThreadsOnThisRank;
-#if GMX_MPI
-    if (PAR(cr) || MULTISIM(cr))
+    /* Avoid MPI calls with uninitialized thread-MPI communicators */
+    if (comm.size_ > 1)
     {
+#if GMX_MPI
         /* Count the threads within this physical node */
-        MPI_Comm_size(cr->mpi_comm_physicalnode, &numRanksOnThisNode);
-        MPI_Allreduce(&numThreadsOnThisRank, &numThreadsOnThisNode, 1, MPI_INT, MPI_SUM, cr->mpi_comm_physicalnode);
-    }
+        MPI_Allreduce(&numThreadsOnThisRank, &numThreadsOnThisNode, 1, MPI_INT, MPI_SUM, comm.comm_);
 #endif
+    }
 
     if (numThreadsOnThisNode > hwTop.machine().logicalProcessorCount)
     {
         std::string mesg = "WARNING: ";
         if (GMX_LIB_MPI)
         {
-            mesg += gmx::formatString("On rank %d: o", cr->sim_nodeid);
+            mesg += formatString("On rank %d: o", rank);
         }
         else
         {
             mesg += "O";
         }
-        mesg     += gmx::formatString("versubscribing the available %d logical CPU cores", hwTop.machine().logicalProcessorCount);
+        mesg     += formatString("versubscribing the available %d logical CPU cores", hwTop.machine().logicalProcessorCount);
         if (GMX_LIB_MPI)
         {
             mesg += " per node";
         }
-        mesg     += gmx::formatString(" with %d ", numThreadsOnThisNode);
+        mesg     += formatString(" with %d ", numThreadsOnThisNode);
         if (numRanksOnThisNode == numThreadsOnThisNode)
         {
             if (GMX_THREAD_MPI)
@@ -939,3 +945,5 @@ void checkHardwareOversubscription(int                          numThreadsOnThis
         GMX_LOG(mdlog.warning).asParagraph().appendTextFormatted(mesg.c_str());
     }
 }
+
+} // namespace