From 31cc5ae98345cbc1039915ac383a5ef5a567a1b0 Mon Sep 17 00:00:00 2001
From: Berk Hess <hess@kth.se>
Date: Tue, 19 May 2015 11:10:06 +0200
Subject: [PATCH] Don't use PME ranks with GPUs and -npme=-1

The code disabling the automated PME rank choice with GPUs was
accidentally moved after init_domain_decomposition. This caused
PME ranks to be set up, but later a fatal_error occured for
inconsistent PP rank and GPU counts.
Refs #1374.

Change-Id: I5f6bcc90fecac7f63b332b8f1acca7368b5f71bc
---
 src/programs/mdrun/runner.c | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/programs/mdrun/runner.c b/src/programs/mdrun/runner.c
index b1e52db1d7..744e9f501e 100644
--- a/src/programs/mdrun/runner.c
+++ b/src/programs/mdrun/runner.c
@@ -1328,7 +1328,13 @@ int mdrunner(gmx_hw_opt_t *hw_opt,
     {
         /* now broadcast everything to the non-master nodes/threads: */
         init_parallel(cr, inputrec, mtop);
+
+        /* The master rank decided on the use of GPUs,
+         * broadcast this information to all ranks.
+         */
+        gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
     }
+
     if (fplog != NULL)
     {
         pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
@@ -1379,6 +1385,15 @@ int mdrunner(gmx_hw_opt_t *hw_opt,
         cr->npmenodes = 0;
     }
 
+    if (bUseGPU && cr->npmenodes < 0)
+    {
+        /* With GPUs we don't automatically use PME-only ranks. PME ranks can
+         * improve performance with many threads per GPU, since our OpenMP
+         * scaling is bad, but it's difficult to automate the setup.
+         */
+        cr->npmenodes = 0;
+    }
+
 #ifdef GMX_FAHCORE
     if (MASTER(cr))
     {
@@ -1547,22 +1562,8 @@ int mdrunner(gmx_hw_opt_t *hw_opt,
                           (cr->duty & DUTY_PP) == 0,
                           inputrec->cutoff_scheme == ecutsVERLET);
 
-    if (PAR(cr))
-    {
-        /* The master rank decided on the use of GPUs,
-         * broadcast this information to all ranks.
-         */
-        gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
-    }
-
     if (bUseGPU)
     {
-        if (cr->npmenodes == -1)
-        {
-            /* Don't automatically use PME-only nodes with GPUs */
-            cr->npmenodes = 0;
-        }
-
         /* Select GPU id's to use */
         gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU,
                            &hw_opt->gpu_opt);
-- 
2.22.0