enabled 1 PP + 1 PME node

Change-Id: I18a4c2bac71f1b5b81d9d374b212bfb9edc7a1e8

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index cb425bf5ea880fd2a3ccac35035ebc9ff37c74f8..e8de945b94ed92b198368896fec394f7ba5f441e 100644 (file)
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -480,6 +480,11 @@ static const ivec dd_zp2[dd_zp2n] = {{0, 0, 4}, {1, 3, 4}};
 #define dd_zp1n 1
 static const ivec dd_zp1[dd_zp1n] = {{0, 0, 2}};
 
+/* The 0D setup */
+#define dd_z0n  1
+#define dd_zp0n 1
+static const ivec dd_zp0[dd_zp0n] = {{0, 0, 1}};
+
 /* Factors used to avoid problems due to rounding issues */
 #define DD_CELL_MARGIN       1.0001
 #define DD_CELL_MARGIN2      1.00005
@@ -5279,6 +5284,13 @@ static void get_load_distribution(gmx_domdec_t *dd, gmx_wallcycle_t wcycle)
 
     bSepPME = (dd->pme_nodeid >= 0);
 
+    if (dd->ndim == 0 && bSepPME)
+    {
+        /* Without decomposition, but with PME nodes, we need the load */
+        comm->load[0].mdf = comm->cycl[ddCyclPPduringPME];
+        comm->load[0].pme = comm->cycl[ddCyclPME];
+    }
+
     for (d = dd->ndim-1; d >= 0; d--)
     {
         dim = dd->dim[d];
@@ -5457,7 +5469,7 @@ static void print_dd_load_av(FILE *fplog, gmx_domdec_t *dd)
 {
     char               buf[STRLEN];
     int                npp, npme, nnodes, d, limp;
-    float              imbal, pme_f_ratio, lossf, lossp = 0;
+    float              imbal, pme_f_ratio, lossf = 0, lossp = 0;
     gmx_bool           bLim;
     gmx_domdec_comm_t *comm;
 
@@ -5467,15 +5479,18 @@ static void print_dd_load_av(FILE *fplog, gmx_domdec_t *dd)
         npp    = dd->nnodes;
         npme   = (dd->pme_nodeid >= 0) ? comm->npmenodes : 0;
         nnodes = npp + npme;
-        imbal  = comm->load_max*npp/comm->load_sum - 1;
-        lossf  = dd_force_imb_perf_loss(dd);
-        sprintf(buf, " Average load imbalance: %.1f %%\n", imbal*100);
-        fprintf(fplog, "%s", buf);
-        fprintf(stderr, "\n");
-        fprintf(stderr, "%s", buf);
-        sprintf(buf, " Part of the total run time spent waiting due to load imbalance: %.1f %%\n", lossf*100);
-        fprintf(fplog, "%s", buf);
-        fprintf(stderr, "%s", buf);
+        if (dd->nnodes > 1)
+        {
+            imbal  = comm->load_max*npp/comm->load_sum - 1;
+            lossf  = dd_force_imb_perf_loss(dd);
+            sprintf(buf, " Average load imbalance: %.1f %%\n", imbal*100);
+            fprintf(fplog, "%s", buf);
+            fprintf(stderr, "\n");
+            fprintf(stderr, "%s", buf);
+            sprintf(buf, " Part of the total run time spent waiting due to load imbalance: %.1f %%\n", lossf*100);
+            fprintf(fplog, "%s", buf);
+            fprintf(stderr, "%s", buf);
+        }
         bLim = FALSE;
         if (comm->bDynLoadBal)
         {
@@ -5600,7 +5615,10 @@ static void dd_print_load(FILE *fplog, gmx_domdec_t *dd, gmx_int64_t step)
         fprintf(fplog, "  vol min/aver %5.3f%c",
                 dd_vol_min(dd), flags ? '!' : ' ');
     }
-    fprintf(fplog, " load imb.: force %4.1f%%", dd_f_imbal(dd)*100);
+    if (dd->nnodes > 1)
+    {
+        fprintf(fplog, " load imb.: force %4.1f%%", dd_f_imbal(dd)*100);
+    }
     if (dd->comm->cycl_n[ddCyclPME])
     {
         fprintf(fplog, "  pme mesh/force %5.3f", dd_pme_f_ratio(dd));
@@ -5615,7 +5633,10 @@ static void dd_print_load_verbose(gmx_domdec_t *dd)
         fprintf(stderr, "vol %4.2f%c ",
                 dd_vol_min(dd), dd_load_flags(dd) ? '!' : ' ');
     }
-    fprintf(stderr, "imb F %2d%% ", (int)(dd_f_imbal(dd)*100+0.5));
+    if (dd->nnodes > 1)
+    {
+        fprintf(stderr, "imb F %2d%% ", (int)(dd_f_imbal(dd)*100+0.5));
+    }
     if (dd->comm->cycl_n[ddCyclPME])
     {
         fprintf(stderr, "pme/F %4.2f ", dd_pme_f_ratio(dd));
@@ -5747,8 +5768,13 @@ static void make_load_communicators(gmx_domdec_t gmx_unused *dd)
         fprintf(debug, "Making load communicators\n");
     }
 
-    snew(dd->comm->load, dd->ndim);
-    snew(dd->comm->mpi_comm_load, dd->ndim);
+    snew(dd->comm->load,          std::max(dd->ndim, 1));
+    snew(dd->comm->mpi_comm_load, std::max(dd->ndim, 1));
+
+    if (dd->ndim == 0)
+    {
+        return;
+    }
 
     clear_ivec(loc);
     make_load_communicator(dd, 0, loc);
@@ -5843,6 +5869,14 @@ void setup_dd_grid(FILE *fplog, gmx_domdec_t *dd)
                 copy_ivec(dd_zp1[i], dd_zp[i]);
             }
             break;
+        case 0:
+            nzone  = dd_z0n;
+            nzonep = dd_zp0n;
+            for (i = 0; i < nzonep; i++)
+            {
+                copy_ivec(dd_zp0[i], dd_zp[i]);
+            }
+            break;
         default:
             gmx_fatal(FARGS, "Can only do 1, 2 or 3D domain decomposition");
             nzone  = 0;

diff --git a/src/gromacs/domdec/domdec_network.cpp b/src/gromacs/domdec/domdec_network.cpp
index 3710f598b645b794cb01717a77c4245bf761c480..c7424bfb22b6feb1341a4c791de55e41f0ba06aa 100644 (file)
--- a/src/gromacs/domdec/domdec_network.cpp
+++ b/src/gromacs/domdec/domdec_network.cpp
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2008,2009,2010,2012,2013,2014, by the GROMACS development team, led by
+ * Copyright (c) 2008,2009,2010,2012,2013,2014,2015, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -232,44 +232,61 @@ void dd_sendrecv2_rvec(const gmx_domdec_t gmx_unused *dd,
 void dd_bcast(gmx_domdec_t gmx_unused *dd, int gmx_unused nbytes, void gmx_unused *data)
 {
 #ifdef GMX_MPI
-#ifdef GMX_BLUEGENE
-    if (nbytes > 0)
+    if (dd->nnodes > 1)
     {
+#ifdef GMX_BLUEGENE
+        if (nbytes > 0)
+        {
 #endif
-    MPI_Bcast(data, nbytes, MPI_BYTE,
-              DDMASTERRANK(dd), dd->mpi_comm_all);
+        MPI_Bcast(data, nbytes, MPI_BYTE,
+                  DDMASTERRANK(dd), dd->mpi_comm_all);
 #ifdef GMX_BLUEGENE
-}
+    }
 #endif
+    }
 #endif
 }
 
 void dd_bcastc(gmx_domdec_t *dd, int nbytes, void *src, void *dest)
 {
-    if (DDMASTER(dd))
+    if (DDMASTER(dd) || dd->nnodes == 1)
     {
         memcpy(dest, src, nbytes);
     }
 #ifdef GMX_MPI
-#ifdef GMX_BLUEGENE
-    if (nbytes > 0)
+    if (dd->nnodes > 1)
     {
+#ifdef GMX_BLUEGENE
+        if (nbytes > 0)
+        {
 #endif
-    MPI_Bcast(dest, nbytes, MPI_BYTE,
-              DDMASTERRANK(dd), dd->mpi_comm_all);
+        MPI_Bcast(dest, nbytes, MPI_BYTE,
+                  DDMASTERRANK(dd), dd->mpi_comm_all);
 #ifdef GMX_BLUEGENE
-}
+    }
 #endif
+    }
 #endif
 }
 
-void dd_scatter(gmx_domdec_t gmx_unused *dd, int gmx_unused nbytes, void gmx_unused *src, void gmx_unused *dest)
+void dd_scatter(gmx_domdec_t gmx_unused *dd, int gmx_unused nbytes, void gmx_unused *src, void *dest)
 {
 #ifdef GMX_MPI
-    MPI_Scatter(src, nbytes, MPI_BYTE,
-                dest, nbytes, MPI_BYTE,
-                DDMASTERRANK(dd), dd->mpi_comm_all);
+    if (dd->nnodes > 1)
+    {
+        MPI_Scatter(src, nbytes, MPI_BYTE,
+                    dest, nbytes, MPI_BYTE,
+                    DDMASTERRANK(dd), dd->mpi_comm_all);
+    }
+    else
 #endif
+    {
+        /* 1 rank, either we copy everything, or dest=src: nothing to do */
+        if (dest != src)
+        {
+            memcpy(dest, src, nbytes);
+        }
+    }
 }
 
 void dd_gather(gmx_domdec_t gmx_unused *dd, int gmx_unused nbytes, void gmx_unused *src, void gmx_unused *dest)
@@ -282,21 +299,32 @@ void dd_gather(gmx_domdec_t gmx_unused *dd, int gmx_unused nbytes, void gmx_unus
 }
 
 void dd_scatterv(gmx_domdec_t gmx_unused *dd,
-                 int gmx_unused *scounts, int gmx_unused *disps, void gmx_unused *sbuf,
-                 int gmx_unused rcount, void gmx_unused *rbuf)
+                 int gmx_unused *scounts, int gmx_unused *disps, void *sbuf,
+                 int rcount, void *rbuf)
 {
 #ifdef GMX_MPI
     int dum;
 
-    if (rcount == 0)
+    if (dd->nnodes > 1)
     {
-        /* MPI does not allow NULL pointers */
-        rbuf = &dum;
+        if (rcount == 0)
+        {
+            /* MPI does not allow NULL pointers */
+            rbuf = &dum;
+        }
+        MPI_Scatterv(sbuf, scounts, disps, MPI_BYTE,
+                     rbuf, rcount, MPI_BYTE,
+                     DDMASTERRANK(dd), dd->mpi_comm_all);
     }
-    MPI_Scatterv(sbuf, scounts, disps, MPI_BYTE,
-                 rbuf, rcount, MPI_BYTE,
-                 DDMASTERRANK(dd), dd->mpi_comm_all);
+    else
 #endif
+    {
+        /* 1 rank, either we copy everything, or rbuf=sbuf: nothing to do */
+        if (rbuf != sbuf)
+        {
+            memcpy(rbuf, sbuf, rcount);
+        }
+    }
 }
 
 void dd_gatherv(gmx_domdec_t gmx_unused *dd,

diff --git a/src/gromacs/domdec/domdec_setup.cpp b/src/gromacs/domdec/domdec_setup.cpp
index 8d0bf79b7b2e6e1d753728f442876b0b3f42c8b4..a6d9a4a2736c67401a1447875dddc058942835f4 100644 (file)
--- a/src/gromacs/domdec/domdec_setup.cpp
+++ b/src/gromacs/domdec/domdec_setup.cpp
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2008,2009,2010,2011,2012,2013,2014, by the GROMACS development team, led by
+ * Copyright (c) 2008,2009,2010,2011,2012,2013,2014,2015, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -721,11 +721,6 @@ real dd_choose_grid(FILE *fplog,
         {
             if (cr->npmenodes > 0)
             {
-                if (cr->nnodes <= 2)
-                {
-                    gmx_fatal(FARGS,
-                              "Cannot have separate PME ranks with 2 or fewer ranks");
-                }
                 if (cr->npmenodes >= cr->nnodes)
                 {
                     gmx_fatal(FARGS,

diff --git a/src/gromacs/mdlib/broadcaststructs.cpp b/src/gromacs/mdlib/broadcaststructs.cpp
index caadeb67a352386fdfe18e1faa1b6275f190b6cf..371e8b7cd50b82164a04e7d7ef2a33cc38f9f094 100644 (file)
--- a/src/gromacs/mdlib/broadcaststructs.cpp
+++ b/src/gromacs/mdlib/broadcaststructs.cpp
@@ -239,7 +239,7 @@ void bcast_state(const t_commrec *cr, t_state *state)
     int      i, nnht, nnhtp;
     gmx_bool bAlloc;
 
-    if (!PAR(cr))
+    if (!PAR(cr) || (cr->nnodes - cr->npmenodes <= 1))
     {
         return;
     }