Enable TPI with the Verlet cut-off scheme

[alexxy/gromacs.git] / src / gromacs / taskassignment / resourcedivision.cpp

diff --git a/src/gromacs/taskassignment/resourcedivision.cpp b/src/gromacs/taskassignment/resourcedivision.cpp
index 452a771616b084ef838a86b486ed57b9ce664dba..a7e7b0a9956764e74f2c3d1c5414b53323572e9c 100644 (file)
--- a/src/gromacs/taskassignment/resourcedivision.cpp
+++ b/src/gromacs/taskassignment/resourcedivision.cpp
@@ -655,7 +655,8 @@ static void print_hw_opt(FILE *fp, const gmx_hw_opt_t *hw_opt)
 void checkAndUpdateHardwareOptions(const gmx::MDLogger &mdlog,
                                    gmx_hw_opt_t        *hw_opt,
                                    const bool           isSimulationMasterRank,
-                                   const int            nPmeRanks)
+                                   const int            nPmeRanks,
+                                   const t_inputrec    *inputrec)
 {
     /* Currently hw_opt only contains default settings or settings supplied
      * by the user on the command line.
@@ -688,6 +689,18 @@ void checkAndUpdateHardwareOptions(const gmx::MDLogger &mdlog,
                       "compiled without thread-MPI");
         }
     }
+
+    /* With thread-MPI we need to handle TPI and #OpenMP-threads=auto early,
+     * so we can parallelize using MPI only. The general check is done later.
+     */
+    if (GMX_THREAD_MPI && isSimulationMasterRank)
+    {
+        GMX_RELEASE_ASSERT(inputrec, "Expect a valid inputrec");
+        if (EI_TPI(inputrec->eI) && hw_opt->nthreads_omp == 0)
+        {
+            hw_opt->nthreads_omp = 1;
+        }
+    }
     /* With thread-MPI the master thread sets hw_opt->totNumThreadsIsAuto.
      * The other threads receive a partially processed hw_opt from the master
      * thread and should not set hw_opt->totNumThreadsIsAuto again.
@@ -801,8 +814,18 @@ void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t         *hw_opt,
                                              const gmx_multisim_t *ms,
                                              int                   numRanksOnThisNode,
                                              PmeRunMode            pmeRunMode,
-                                             const gmx_mtop_t     &mtop)
+                                             const gmx_mtop_t     &mtop,
+                                             const t_inputrec     &inputrec)
 {
+    if (EI_TPI(inputrec.eI))
+    {
+        if (hw_opt->nthreads_omp > 1)
+        {
+            gmx_fatal(FARGS, "You requested OpenMP parallelization, which is not supported with TPI.");
+        }
+        hw_opt->nthreads_omp = 1;
+    }
+
     if (GMX_THREAD_MPI)
     {