Require explicit MPI_COMM for gmx_bcast and gmx_barrier

This changes gmx_bcast and gmx_barrier to take the MPI communicator
explicitly instead of taking a pointer to t_commrec and using
mpi_comm_mygroup. This also allows to remove gmx_bcast_sim and leave
the responsibility of passing the right communicator to the caller.

This is a first step in breaking up t_commrec. These functions
are the subset of low-level networking functions which are used
before domain decomposition (and hence PP/PME ranks) is set up.

Refs #2395

diff --git a/src/gromacs/applied_forces/densityfitting.cpp b/src/gromacs/applied_forces/densityfitting.cpp
index 2889e61a5072eabe85e1f4a13fa11f2c8a34ec45..105f6144bf85ee9e0b551e2c2a1814f1483b6a01 100644 (file)
--- a/src/gromacs/applied_forces/densityfitting.cpp
+++ b/src/gromacs/applied_forces/densityfitting.cpp
@@ -454,9 +454,12 @@ public:
     {
         if (PAR(&(checkpointBroadcast.cr_)))
         {
-            block_bc(&(checkpointBroadcast.cr_), densityFittingState_.stepsSinceLastCalculation_);
-            block_bc(&(checkpointBroadcast.cr_), densityFittingState_.adaptiveForceConstantScale_);
-            block_bc(&(checkpointBroadcast.cr_), densityFittingState_.exponentialMovingAverageState_);
+            block_bc(checkpointBroadcast.cr_.mpi_comm_mygroup,
+                     densityFittingState_.stepsSinceLastCalculation_);
+            block_bc(checkpointBroadcast.cr_.mpi_comm_mygroup,
+                     densityFittingState_.adaptiveForceConstantScale_);
+            block_bc(checkpointBroadcast.cr_.mpi_comm_mygroup,
+                     densityFittingState_.exponentialMovingAverageState_);
         }
     }
 

diff --git a/src/gromacs/awh/awh.cpp b/src/gromacs/awh/awh.cpp
index 610f67470c89593975ac0304dfc87caf7666ba50..dfa2f71b967e41e5efc265650d2e46907c8b79cd 100644 (file)
--- a/src/gromacs/awh/awh.cpp
+++ b/src/gromacs/awh/awh.cpp
@@ -323,7 +323,7 @@ void Awh::restoreStateFromHistory(const AwhHistory* awhHistory)
     }
     if (PAR(commRecord_))
     {
-        gmx_bcast(sizeof(potentialOffset_), &potentialOffset_, commRecord_);
+        gmx_bcast(sizeof(potentialOffset_), &potentialOffset_, commRecord_->mpi_comm_mygroup);
     }
 
     for (size_t k = 0; k < biasCoupledToSystem_.size(); k++)

diff --git a/src/gromacs/awh/biasstate.cpp b/src/gromacs/awh/biasstate.cpp
index c75b8639175fe8e89dc122a958eaf5137fe32f3c..8a90b36bc5b9aaca52ebd59fc2caaa7a160c6dd9 100644 (file)
--- a/src/gromacs/awh/biasstate.cpp
+++ b/src/gromacs/awh/biasstate.cpp
@@ -132,7 +132,7 @@ void sumOverSimulations(gmx::ArrayRef<T> arrayRef, const t_commrec* commRecord,
     }
     if (commRecord->nnodes > 1)
     {
-        gmx_bcast(arrayRef.size() * sizeof(T), arrayRef.data(), commRecord);
+        gmx_bcast(arrayRef.size() * sizeof(T), arrayRef.data(), commRecord->mpi_comm_mygroup);
     }
 }
 
@@ -1372,13 +1372,14 @@ void BiasState::restoreFromHistory(const AwhBiasHistory& biasHistory, const Bias
 
 void BiasState::broadcast(const t_commrec* commRecord)
 {
-    gmx_bcast(sizeof(coordState_), &coordState_, commRecord);
+    gmx_bcast(sizeof(coordState_), &coordState_, commRecord->mpi_comm_mygroup);
 
-    gmx_bcast(points_.size() * sizeof(PointState), points_.data(), commRecord);
+    gmx_bcast(points_.size() * sizeof(PointState), points_.data(), commRecord->mpi_comm_mygroup);
 
-    gmx_bcast(weightSumCovering_.size() * sizeof(double), weightSumCovering_.data(), commRecord);
+    gmx_bcast(weightSumCovering_.size() * sizeof(double), weightSumCovering_.data(),
+              commRecord->mpi_comm_mygroup);
 
-    gmx_bcast(sizeof(histogramSize_), &histogramSize_, commRecord);
+    gmx_bcast(sizeof(histogramSize_), &histogramSize_, commRecord->mpi_comm_mygroup);
 }
 
 void BiasState::setFreeEnergyToConvolvedPmf(const std::vector<DimParams>& dimParams, const BiasGrid& grid)

diff --git a/src/gromacs/domdec/box.cpp b/src/gromacs/domdec/box.cpp
index 03b0e9f69a33258bb6fe1eea00303e5726a89c80..9f40425c1bad4ab8a41255635112ab1b2c759cf1 100644 (file)
--- a/src/gromacs/domdec/box.cpp
+++ b/src/gromacs/domdec/box.cpp
@@ -313,5 +313,5 @@ void set_ddbox_cr(const t_commrec&               cr,
                       nullptr, ddbox);
     }
 
-    gmx_bcast(sizeof(gmx_ddbox_t), ddbox, &cr);
+    gmx_bcast(sizeof(gmx_ddbox_t), ddbox, cr.mpi_comm_mygroup);
 }

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index aed2af87cbe5522259c31d9311df501fa50cbae0..bbd5d8a213a89898aff833d094edbb213ede4174 100644 (file)
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -2161,8 +2161,8 @@ static DDSystemInfo getSystemInfo(const gmx::MDLogger&           mdlog,
                 dd_bonded_cg_distance(mdlog, &mtop, &ir, as_rvec_array(xGlobal.data()), box,
                                       options.checkBondedInteractions, &r_2b, &r_mb);
             }
-            gmx_bcast(sizeof(r_2b), &r_2b, cr);
-            gmx_bcast(sizeof(r_mb), &r_mb, cr);
+            gmx_bcast(sizeof(r_2b), &r_2b, cr->mpi_comm_mygroup);
+            gmx_bcast(sizeof(r_mb), &r_mb, cr->mpi_comm_mygroup);
 
             /* We use an initial margin of 10% for the minimum cell size,
              * except when we are just below the non-bonded cut-off.

diff --git a/src/gromacs/domdec/domdec_setup.cpp b/src/gromacs/domdec/domdec_setup.cpp
index a22ef72b091fbda0a2707db25bce5bac54fef8de..c22882f17b61d5a89d5ac123314ea03a52b2414c 100644 (file)
--- a/src/gromacs/domdec/domdec_setup.cpp
+++ b/src/gromacs/domdec/domdec_setup.cpp
@@ -940,10 +940,10 @@ DDGridSetup getDDGridSetup(const gmx::MDLogger&           mdlog,
     }
 
     /* Communicate the information set by the master to all ranks */
-    gmx_bcast(sizeof(numDomains), numDomains, cr);
+    gmx_bcast(sizeof(numDomains), numDomains, cr->mpi_comm_mygroup);
     if (EEL_PME(ir.coulombtype))
     {
-        gmx_bcast(sizeof(numPmeOnlyRanks), &numPmeOnlyRanks, cr);
+        gmx_bcast(sizeof(numPmeOnlyRanks), &numPmeOnlyRanks, cr->mpi_comm_mygroup);
     }
 
     DDGridSetup ddGridSetup;

diff --git a/src/gromacs/essentialdynamics/edsam.cpp b/src/gromacs/essentialdynamics/edsam.cpp
index ba16e34d72453923d7713a3ea09d7341823f5d17..aebd672980e33fa4eadd1843f523a45086755c60 100644 (file)
--- a/src/gromacs/essentialdynamics/edsam.cpp
+++ b/src/gromacs/essentialdynamics/edsam.cpp
@@ -1184,10 +1184,10 @@ static std::unique_ptr<gmx::EssentialDynamics> ed_open(int
 /* Broadcasts the structure data */
 static void bc_ed_positions(const t_commrec* cr, struct gmx_edx* s, EssentialDynamicsStructure stype)
 {
-    snew_bc(cr, s->anrs, s->nr); /* Index numbers     */
-    snew_bc(cr, s->x, s->nr);    /* Positions         */
-    nblock_bc(cr, s->nr, s->anrs);
-    nblock_bc(cr, s->nr, s->x);
+    snew_bc(MASTER(cr), s->anrs, s->nr); /* Index numbers     */
+    snew_bc(MASTER(cr), s->x, s->nr);    /* Positions         */
+    nblock_bc(cr->mpi_comm_mygroup, s->nr, s->anrs);
+    nblock_bc(cr->mpi_comm_mygroup, s->nr, s->x);
 
     /* For the average & reference structures we need an array for the collective indices,
      * and we need to broadcast the masses as well */
@@ -1197,25 +1197,26 @@ static void bc_ed_positions(const t_commrec* cr, struct gmx_edx* s, EssentialDyn
         snew(s->c_ind, s->nr); /* Collective indices */
         /* Local atom indices get assigned in dd_make_local_group_indices.
          * There, also memory is allocated */
-        s->nalloc_loc = 0;            /* allocation size of s->anrs_loc */
-        snew_bc(cr, s->x_old, s->nr); /* To be able to always make the ED molecule whole, ... */
-        nblock_bc(cr, s->nr, s->x_old); /* ... keep track of shift changes with the help of old coords */
+        s->nalloc_loc = 0;                    /* allocation size of s->anrs_loc */
+        snew_bc(MASTER(cr), s->x_old, s->nr); /* To be able to always make the ED molecule whole, ... */
+        nblock_bc(cr->mpi_comm_mygroup, s->nr,
+                  s->x_old); /* ... keep track of shift changes with the help of old coords */
     }
 
     /* broadcast masses for the reference structure (for mass-weighted fitting) */
     if (stype == EssentialDynamicsStructure::Reference)
     {
-        snew_bc(cr, s->m, s->nr);
-        nblock_bc(cr, s->nr, s->m);
+        snew_bc(MASTER(cr), s->m, s->nr);
+        nblock_bc(cr->mpi_comm_mygroup, s->nr, s->m);
     }
 
     /* For the average structure we might need the masses for mass-weighting */
     if (stype == EssentialDynamicsStructure::Average)
     {
-        snew_bc(cr, s->sqrtm, s->nr);
-        nblock_bc(cr, s->nr, s->sqrtm);
-        snew_bc(cr, s->m, s->nr);
-        nblock_bc(cr, s->nr, s->m);
+        snew_bc(MASTER(cr), s->sqrtm, s->nr);
+        nblock_bc(cr->mpi_comm_mygroup, s->nr, s->sqrtm);
+        snew_bc(MASTER(cr), s->m, s->nr);
+        nblock_bc(cr->mpi_comm_mygroup, s->nr, s->m);
     }
 }
 
@@ -1225,23 +1226,23 @@ static void bc_ed_vecs(const t_commrec* cr, t_eigvec* ev, int length)
 {
     int i;
 
-    snew_bc(cr, ev->ieig, ev->neig);    /* index numbers of eigenvector  */
-    snew_bc(cr, ev->stpsz, ev->neig);   /* stepsizes per eigenvector     */
-    snew_bc(cr, ev->xproj, ev->neig);   /* instantaneous x projection    */
-    snew_bc(cr, ev->fproj, ev->neig);   /* instantaneous f projection    */
-    snew_bc(cr, ev->refproj, ev->neig); /* starting or target projection */
+    snew_bc(MASTER(cr), ev->ieig, ev->neig);    /* index numbers of eigenvector  */
+    snew_bc(MASTER(cr), ev->stpsz, ev->neig);   /* stepsizes per eigenvector     */
+    snew_bc(MASTER(cr), ev->xproj, ev->neig);   /* instantaneous x projection    */
+    snew_bc(MASTER(cr), ev->fproj, ev->neig);   /* instantaneous f projection    */
+    snew_bc(MASTER(cr), ev->refproj, ev->neig); /* starting or target projection */
 
-    nblock_bc(cr, ev->neig, ev->ieig);
-    nblock_bc(cr, ev->neig, ev->stpsz);
-    nblock_bc(cr, ev->neig, ev->xproj);
-    nblock_bc(cr, ev->neig, ev->fproj);
-    nblock_bc(cr, ev->neig, ev->refproj);
+    nblock_bc(cr->mpi_comm_mygroup, ev->neig, ev->ieig);
+    nblock_bc(cr->mpi_comm_mygroup, ev->neig, ev->stpsz);
+    nblock_bc(cr->mpi_comm_mygroup, ev->neig, ev->xproj);
+    nblock_bc(cr->mpi_comm_mygroup, ev->neig, ev->fproj);
+    nblock_bc(cr->mpi_comm_mygroup, ev->neig, ev->refproj);
 
-    snew_bc(cr, ev->vec, ev->neig); /* Eigenvector components        */
+    snew_bc(MASTER(cr), ev->vec, ev->neig); /* Eigenvector components        */
     for (i = 0; i < ev->neig; i++)
     {
-        snew_bc(cr, ev->vec[i], length);
-        nblock_bc(cr, length, ev->vec[i]);
+        snew_bc(MASTER(cr), ev->vec[i], length);
+        nblock_bc(cr->mpi_comm_mygroup, length, ev->vec[i]);
     }
 }
 
@@ -1251,17 +1252,17 @@ static void bc_ed_vecs(const t_commrec* cr, t_eigvec* ev, int length)
 static void broadcast_ed_data(const t_commrec* cr, gmx_edsam* ed)
 {
     /* Master lets the other nodes know if its ED only or also flooding */
-    gmx_bcast(sizeof(ed->eEDtype), &(ed->eEDtype), cr);
+    gmx_bcast(sizeof(ed->eEDtype), &(ed->eEDtype), cr->mpi_comm_mygroup);
 
     int numedis = ed->edpar.size();
     /* First let everybody know how many ED data sets to expect */
-    gmx_bcast(sizeof(numedis), &numedis, cr);
-    nblock_abc(cr, numedis, &(ed->edpar));
+    gmx_bcast(sizeof(numedis), &numedis, cr->mpi_comm_mygroup);
+    nblock_abc(MASTER(cr), cr->mpi_comm_mygroup, numedis, &(ed->edpar));
     /* Now transfer the ED data set(s) */
     for (auto& edi : ed->edpar)
     {
         /* Broadcast a single ED data set */
-        block_bc(cr, edi);
+        block_bc(cr->mpi_comm_mygroup, edi);
 
         /* Broadcast positions */
         bc_ed_positions(cr, &(edi.sref),
@@ -1284,10 +1285,10 @@ static void broadcast_ed_data(const t_commrec* cr, gmx_edsam* ed)
         /* For harmonic restraints the reference projections can change with time */
         if (edi.flood.bHarmonic)
         {
-            snew_bc(cr, edi.flood.initialReferenceProjection, edi.flood.vecs.neig);
-            snew_bc(cr, edi.flood.referenceProjectionSlope, edi.flood.vecs.neig);
-            nblock_bc(cr, edi.flood.vecs.neig, edi.flood.initialReferenceProjection);
-            nblock_bc(cr, edi.flood.vecs.neig, edi.flood.referenceProjectionSlope);
+            snew_bc(MASTER(cr), edi.flood.initialReferenceProjection, edi.flood.vecs.neig);
+            snew_bc(MASTER(cr), edi.flood.referenceProjectionSlope, edi.flood.vecs.neig);
+            nblock_bc(cr->mpi_comm_mygroup, edi.flood.vecs.neig, edi.flood.initialReferenceProjection);
+            nblock_bc(cr->mpi_comm_mygroup, edi.flood.vecs.neig, edi.flood.referenceProjectionSlope);
         }
     }
 }
@@ -3029,7 +3030,7 @@ std::unique_ptr<gmx::EssentialDynamics> init_edsam(const gmx::MDLogger&        m
     for (auto edi = ed->edpar.begin(); edi != ed->edpar.end(); ++edi)
     {
         /* Allocate space for ED buffer variables */
-        snew_bc(cr, edi->buf, 1); /* MASTER has already allocated edi->buf in init_edi() */
+        snew_bc(MASTER(cr), edi->buf, 1); /* MASTER has already allocated edi->buf in init_edi() */
         snew(edi->buf->do_edsam, 1);
 
         /* Space for collective ED buffer variables */

diff --git a/src/gromacs/fileio/checkpoint.cpp b/src/gromacs/fileio/checkpoint.cpp
index 72f8a8dea7456b689f26967930970d36f14a9cbb..ee9c72daa085c116feea500780de38dfbec228be 100644 (file)
--- a/src/gromacs/fileio/checkpoint.cpp
+++ b/src/gromacs/fileio/checkpoint.cpp
@@ -2837,7 +2837,7 @@ void load_checkpoint(const char*                   fn,
     }
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(headerContents.step), &headerContents.step, cr);
+        gmx_bcast(sizeof(headerContents.step), &headerContents.step, cr->mpi_comm_mygroup);
         gmx::MdModulesCheckpointReadingBroadcast broadcastCheckPointData = { *cr, headerContents.file_version };
         mdModulesNotifier.checkpointingNotifications_.notify(broadcastCheckPointData);
     }

diff --git a/src/gromacs/gmxlib/network.cpp b/src/gromacs/gmxlib/network.cpp
index d75d7faa96c97e00b661c6643ce5e028c95a1f1d..3670c866acd538e7cc886fd0f2a420d55b84d9c1 100644 (file)
--- a/src/gromacs/gmxlib/network.cpp
+++ b/src/gromacs/gmxlib/network.cpp
@@ -239,30 +239,21 @@ void gmx_setup_nodecomm(FILE gmx_unused* fplog, t_commrec* cr)
 #endif
 }
 
-void gmx_barrier(const t_commrec gmx_unused* cr)
+void gmx_barrier(MPI_Comm gmx_unused communicator)
 {
 #if !GMX_MPI
     gmx_call("gmx_barrier");
 #else
-    MPI_Barrier(cr->mpi_comm_mygroup);
+    MPI_Barrier(communicator);
 #endif
 }
 
-void gmx_bcast(int gmx_unused nbytes, void gmx_unused* b, const t_commrec gmx_unused* cr)
+void gmx_bcast(int gmx_unused nbytes, void gmx_unused* b, MPI_Comm gmx_unused communicator)
 {
 #if !GMX_MPI
-    gmx_call("gmx_bast");
+    gmx_call("gmx_bcast");
 #else
-    MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mygroup);
-#endif
-}
-
-void gmx_bcast_sim(int gmx_unused nbytes, void gmx_unused* b, const t_commrec gmx_unused* cr)
-{
-#if !GMX_MPI
-    gmx_call("gmx_bast");
-#else
-    MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mysim);
+    MPI_Bcast(b, nbytes, MPI_BYTE, 0, communicator);
 #endif
 }
 

diff --git a/src/gromacs/gmxlib/network.h b/src/gromacs/gmxlib/network.h
index d5d7a7f1910532fb50ce1b147591296b5cd3682a..c17289a4ce7b8b3ecc000a0b8f73cdfe9b711300 100644 (file)
--- a/src/gromacs/gmxlib/network.h
+++ b/src/gromacs/gmxlib/network.h
@@ -76,14 +76,11 @@ void gmx_fill_commrec_from_mpi(t_commrec* cr);
 void gmx_setup_nodecomm(FILE* fplog, struct t_commrec* cr);
 /* Sets up fast global communication for clusters with multi-core nodes */
 
-void gmx_barrier(const struct t_commrec* cr);
-/* Wait till all processes in cr->mpi_comm_mygroup have reached the barrier */
+//! Wait until all processes in communicator have reached the barrier
+void gmx_barrier(MPI_Comm communicator);
 
-void gmx_bcast(int nbytes, void* b, const struct t_commrec* cr);
-/* Broadcast nbytes bytes from the master to cr->mpi_comm_mygroup */
-
-void gmx_bcast_sim(int nbytes, void* b, const struct t_commrec* cr);
-/* Broadcast nbytes bytes from the sim master to cr->mpi_comm_mysim */
+//! Broadcast nbytes bytes from the master to communicator
+void gmx_bcast(int nbytes, void* b, MPI_Comm communicator);
 
 void gmx_sumi(int nr, int r[], const struct t_commrec* cr);
 /* Calculate the global sum of an array of ints */

diff --git a/src/gromacs/imd/imd.cpp b/src/gromacs/imd/imd.cpp
index 4f54a989d3247e7bb01d664e1aa3007a6623b694..9c84d7718b6445cbe7ab4206f29597d2829ce2f7 100644 (file)
--- a/src/gromacs/imd/imd.cpp
+++ b/src/gromacs/imd/imd.cpp
@@ -76,6 +76,7 @@
 #include "gromacs/mdtypes/inputrec.h"
 #include "gromacs/mdtypes/md_enums.h"
 #include "gromacs/mdtypes/mdrunoptions.h"
+#include "gromacs/mdtypes/commrec.h"
 #include "gromacs/mdtypes/state.h"
 #include "gromacs/pbcutil/pbc.h"
 #include "gromacs/timing/wallcycle.h"
@@ -833,7 +834,7 @@ void ImdSession::Impl::syncNodes(const t_commrec* cr, double t)
     /* Notify the other nodes whether we are still connected. */
     if (PAR(cr))
     {
-        block_bc(cr, bConnected);
+        block_bc(cr->mpi_comm_mygroup, bConnected);
     }
 
     /* ...if not connected, the job is done here. */
@@ -845,7 +846,7 @@ void ImdSession::Impl::syncNodes(const t_commrec* cr, double t)
     /* Let the other nodes know whether we got a new IMD synchronization frequency. */
     if (PAR(cr))
     {
-        block_bc(cr, nstimd_new);
+        block_bc(cr->mpi_comm_mygroup, nstimd_new);
     }
 
     /* Now we all set the (new) nstimd communication time step */
@@ -876,7 +877,7 @@ void ImdSession::Impl::syncNodes(const t_commrec* cr, double t)
     /* make new_forces known to the clients */
     if (PAR(cr))
     {
-        block_bc(cr, new_nforces);
+        block_bc(cr->mpi_comm_mygroup, new_nforces);
     }
 
     /* When new_natoms < 0 then we know that these are still the same forces
@@ -909,8 +910,8 @@ void ImdSession::Impl::syncNodes(const t_commrec* cr, double t)
     /* In parallel mode we communicate the to-be-applied forces to the other nodes */
     if (PAR(cr))
     {
-        nblock_bc(cr, nforces, f_ind);
-        nblock_bc(cr, nforces, f);
+        nblock_bc(cr->mpi_comm_mygroup, nforces, f_ind);
+        nblock_bc(cr->mpi_comm_mygroup, nforces, f);
     }
 
     /* done communicating the forces, reset bNewForces */
@@ -1268,7 +1269,7 @@ void ImdSession::Impl::prepareForPositionAssembly(const t_commrec* cr, const rve
     /* Communicate initial coordinates xa_old to all processes */
     if (PAR(cr))
     {
-        gmx_bcast(nat * sizeof(xa_old[0]), xa_old, cr);
+        gmx_bcast(nat * sizeof(xa_old[0]), xa_old, cr->mpi_comm_mygroup);
     }
 }
 
@@ -1373,7 +1374,7 @@ std::unique_ptr<ImdSession> makeImdSession(const t_inputrec*           ir,
     /* Let the other nodes know whether we want IMD */
     if (PAR(cr))
     {
-        block_bc(cr, createSession);
+        block_bc(cr->mpi_comm_mygroup, createSession);
     }
 
     /*... if not we are done.*/
@@ -1472,7 +1473,7 @@ std::unique_ptr<ImdSession> makeImdSession(const t_inputrec*           ir,
     /* do we allow interactive pulling? If so let the other nodes know. */
     if (PAR(cr))
     {
-        block_bc(cr, impl->bForceActivated);
+        block_bc(cr->mpi_comm_mygroup, impl->bForceActivated);
     }
 
     /* setup the listening socket on master process */

diff --git a/src/gromacs/listed_forces/disre.cpp b/src/gromacs/listed_forces/disre.cpp
index 2af56455081939e2045934896b751a9988c78ae3..9eebef862dc5433231c83a1159b909608716e5f6 100644 (file)
--- a/src/gromacs/listed_forces/disre.cpp
+++ b/src/gromacs/listed_forces/disre.cpp
@@ -234,7 +234,7 @@ void init_disres(FILE*                 fplog,
         {
             check_multi_int(fplog, ms, dd->nsystems, "the number of systems per ensemble", FALSE);
         }
-        gmx_bcast_sim(sizeof(int), &dd->nsystems, cr);
+        gmx_bcast(sizeof(int), &dd->nsystems, cr->mpi_comm_mysim);
 
         /* We use to allow any value of nsystems which was a divisor
          * of ms->nsim. But this required an extra communicator which
@@ -402,7 +402,7 @@ void calc_disres_R_6(const t_commrec*      cr,
 
         if (DOMAINDECOMP(cr))
         {
-            gmx_bcast(2 * dd->nres, dd->Rt_6, cr);
+            gmx_bcast(2 * dd->nres, dd->Rt_6, cr->mpi_comm_mygroup);
         }
     }
 

diff --git a/src/gromacs/mdlib/boxdeformation.cpp b/src/gromacs/mdlib/boxdeformation.cpp
index 6adb621b64697718b315ae1f29c2b45351101349..457f0551e02f8794c1a6c3e1a1ff5686f8ea9423 100644 (file)
--- a/src/gromacs/mdlib/boxdeformation.cpp
+++ b/src/gromacs/mdlib/boxdeformation.cpp
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -86,7 +86,7 @@ std::unique_ptr<BoxDeformation> prepareBoxDeformation(const matrix&     initialB
     }
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(box), box, cr);
+        gmx_bcast(sizeof(box), box, cr->mpi_comm_mygroup);
     }
 
     return std::make_unique<BoxDeformation>(inputrec.delta_t, inputrec.init_step, inputrec.deform, box);

diff --git a/src/gromacs/mdlib/broadcaststructs.cpp b/src/gromacs/mdlib/broadcaststructs.cpp
index 93d635fbc60e84d5c7ff78a787b3a70528f10e40..4ab660a823ae573eabdb965cd87143a319bad844 100644 (file)
--- a/src/gromacs/mdlib/broadcaststructs.cpp
+++ b/src/gromacs/mdlib/broadcaststructs.cpp
@@ -41,24 +41,27 @@
 #include "broadcaststructs.h"
 
 #include "gromacs/fileio/tpxio.h"
-#include "gromacs/gmxlib/network.h"
-#include "gromacs/mdtypes/commrec.h"
 #include "gromacs/mdtypes/state.h"
 
 template<typename AllocatorType>
-static void bcastPaddedRVecVector(const t_commrec* cr, gmx::PaddedVector<gmx::RVec, AllocatorType>* v, int numAtoms)
+static void bcastPaddedRVecVector(MPI_Comm                                     communicator,
+                                  gmx::PaddedVector<gmx::RVec, AllocatorType>* v,
+                                  int                                          numAtoms)
 {
     v->resizeWithPadding(numAtoms);
-    nblock_bc(cr, makeArrayRef(*v));
+    nblock_bc(communicator, makeArrayRef(*v));
 }
 
-void broadcastStateWithoutDynamics(const t_commrec* cr, t_state* state)
+void broadcastStateWithoutDynamics(MPI_Comm communicator,
+                                   bool     useDomainDecomposition,
+                                   bool     isParallelRun,
+                                   t_state* state)
 {
-    GMX_RELEASE_ASSERT(!DOMAINDECOMP(cr),
+    GMX_RELEASE_ASSERT(!useDomainDecomposition,
                        "broadcastStateWithoutDynamics should only be used for special cases "
                        "without domain decomposition");
 
-    if (!PAR(cr))
+    if (!isParallelRun)
     {
         return;
     }
@@ -66,8 +69,8 @@ void broadcastStateWithoutDynamics(const t_commrec* cr, t_state* state)
     /* Broadcasts the state sizes and flags from the master to all ranks
      * in cr->mpi_comm_mygroup.
      */
-    block_bc(cr, state->natoms);
-    block_bc(cr, state->flags);
+    block_bc(communicator, state->natoms);
+    block_bc(communicator, state->flags);
 
     for (int i = 0; i < estNR; i++)
     {
@@ -75,10 +78,10 @@ void broadcastStateWithoutDynamics(const t_commrec* cr, t_state* state)
         {
             switch (i)
             {
-                case estLAMBDA: nblock_bc(cr, efptNR, state->lambda.data()); break;
-                case estFEPSTATE: block_bc(cr, state->fep_state); break;
-                case estBOX: block_bc(cr, state->box); break;
-                case estX: bcastPaddedRVecVector(cr, &state->x, state->natoms); break;
+                case estLAMBDA: nblock_bc(communicator, efptNR, state->lambda.data()); break;
+                case estFEPSTATE: block_bc(communicator, state->fep_state); break;
+                case estBOX: block_bc(communicator, state->box); break;
+                case estX: bcastPaddedRVecVector(communicator, &state->x, state->natoms); break;
                 default:
                     GMX_RELEASE_ASSERT(false,
                                        "The state has a dynamic entry, while no dynamic entries "
@@ -89,37 +92,41 @@ void broadcastStateWithoutDynamics(const t_commrec* cr, t_state* state)
     }
 }
 
-static void bc_tpxheader(const t_commrec* cr, TpxFileHeader* tpx)
+static void bc_tpxheader(MPI_Comm communicator, TpxFileHeader* tpx)
 {
-    block_bc(cr, tpx->bIr);
-    block_bc(cr, tpx->bBox);
-    block_bc(cr, tpx->bTop);
-    block_bc(cr, tpx->bX);
-    block_bc(cr, tpx->bV);
-    block_bc(cr, tpx->bF);
-    block_bc(cr, tpx->natoms);
-    block_bc(cr, tpx->ngtc);
-    block_bc(cr, tpx->lambda);
-    block_bc(cr, tpx->fep_state);
-    block_bc(cr, tpx->sizeOfTprBody);
-    block_bc(cr, tpx->fileVersion);
-    block_bc(cr, tpx->fileGeneration);
-    block_bc(cr, tpx->isDouble);
+    block_bc(communicator, tpx->bIr);
+    block_bc(communicator, tpx->bBox);
+    block_bc(communicator, tpx->bTop);
+    block_bc(communicator, tpx->bX);
+    block_bc(communicator, tpx->bV);
+    block_bc(communicator, tpx->bF);
+    block_bc(communicator, tpx->natoms);
+    block_bc(communicator, tpx->ngtc);
+    block_bc(communicator, tpx->lambda);
+    block_bc(communicator, tpx->fep_state);
+    block_bc(communicator, tpx->sizeOfTprBody);
+    block_bc(communicator, tpx->fileVersion);
+    block_bc(communicator, tpx->fileGeneration);
+    block_bc(communicator, tpx->isDouble);
 }
 
-static void bc_tprCharBuffer(const t_commrec* cr, std::vector<char>* charBuffer)
+static void bc_tprCharBuffer(MPI_Comm communicator, bool isMasterRank, std::vector<char>* charBuffer)
 {
     int elements = charBuffer->size();
-    block_bc(cr, elements);
+    block_bc(communicator, elements);
 
-    nblock_abc(cr, elements, charBuffer);
+    nblock_abc(isMasterRank, communicator, elements, charBuffer);
 }
 
-void init_parallel(t_commrec* cr, t_inputrec* inputrec, gmx_mtop_t* mtop, PartialDeserializedTprFile* partialDeserializedTpr)
+void init_parallel(MPI_Comm                    communicator,
+                   bool                        isMasterRank,
+                   t_inputrec*                 inputrec,
+                   gmx_mtop_t*                 mtop,
+                   PartialDeserializedTprFile* partialDeserializedTpr)
 {
-    bc_tpxheader(cr, &partialDeserializedTpr->header);
-    bc_tprCharBuffer(cr, &partialDeserializedTpr->body);
-    if (!MASTER(cr))
+    bc_tpxheader(communicator, &partialDeserializedTpr->header);
+    bc_tprCharBuffer(communicator, isMasterRank, &partialDeserializedTpr->body);
+    if (!isMasterRank)
     {
         completeTprDeserialization(partialDeserializedTpr, inputrec, mtop);
     }

diff --git a/src/gromacs/mdlib/broadcaststructs.h b/src/gromacs/mdlib/broadcaststructs.h
index 877b39d31a83a3c9a47a78ed83d1e3adbbf27384..d6a5b15a264d38a9d9188db5381af12240de9044 100644 (file)
--- a/src/gromacs/mdlib/broadcaststructs.h
+++ b/src/gromacs/mdlib/broadcaststructs.h
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2016,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2016,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -48,69 +48,70 @@
 #include <vector>
 
 #include "gromacs/gmxlib/network.h"
-#include "gromacs/mdtypes/commrec.h"
 #include "gromacs/utility/arrayref.h"
-#include "gromacs/utility/smalloc.h"
 
 struct gmx_mtop_t;
-struct t_commrec;
 struct t_inputrec;
 struct PartialDeserializedTprFile;
 class t_state;
 
-//! Convenience wrapper for gmx_bcast of a single value.
+//! Convenience wrapper for gmx_bcast to communicator of a single value.
 template<typename T>
-void block_bc(const t_commrec* cr, T& data)
+void block_bc(MPI_Comm communicator, T& data)
 {
-    gmx_bcast(sizeof(T), static_cast<void*>(&data), cr);
+    gmx_bcast(sizeof(T), static_cast<void*>(&data), communicator);
 }
-//! Convenience wrapper for gmx_bcast of a C-style array.
+//! Convenience wrapper for gmx_bcast to communicator of a C-style array.
 template<typename T>
-void nblock_bc(const t_commrec* cr, int numElements, T* data)
+void nblock_bc(MPI_Comm communicator, int numElements, T* data)
 {
-    gmx_bcast(numElements * sizeof(T), static_cast<void*>(data), cr);
+    gmx_bcast(numElements * sizeof(T), static_cast<void*>(data), communicator);
 }
-//! Convenience wrapper for gmx_bcast of an ArrayRef<T>
+//! Convenience wrapper for gmx_bcast to communicator of an ArrayRef<T>
 template<typename T>
-void nblock_bc(const t_commrec* cr, gmx::ArrayRef<T> data)
+void nblock_bc(MPI_Comm communicator, gmx::ArrayRef<T> data)
 {
-    gmx_bcast(data.size() * sizeof(T), static_cast<void*>(data.data()), cr);
+    gmx_bcast(data.size() * sizeof(T), static_cast<void*>(data.data()), communicator);
 }
 //! Convenience wrapper for allocation with snew of vectors that need allocation on non-master ranks.
 template<typename T>
-void snew_bc(const t_commrec* cr, T*& data, int numElements)
+void snew_bc(bool isMasterRank, T*& data, int numElements)
 {
-    if (!MASTER(cr))
+    if (!isMasterRank)
     {
         snew(data, numElements);
     }
 }
 //! Convenience wrapper for gmx_bcast of a C-style array which needs allocation on non-master ranks.
 template<typename T>
-void nblock_abc(const t_commrec* cr, int numElements, T** v)
+void nblock_abc(bool isMasterRank, MPI_Comm communicator, int numElements, T** v)
 {
-    snew_bc(cr, v, numElements);
-    nblock_bc(cr, numElements, *v);
+    snew_bc(isMasterRank, v, numElements);
+    nblock_bc(communicator, numElements, *v);
 }
 //! Convenience wrapper for gmx_bcast of a std::vector which needs resizing on non-master ranks.
 template<typename T>
-void nblock_abc(const t_commrec* cr, int numElements, std::vector<T>* v)
+void nblock_abc(bool isMasterRank, MPI_Comm communicator, int numElements, std::vector<T>* v)
 {
-    if (!MASTER(cr))
+    if (!isMasterRank)
     {
         v->resize(numElements);
     }
-    gmx_bcast(numElements * sizeof(T), v->data(), cr);
+    gmx_bcast(numElements * sizeof(T), v->data(), communicator);
 }
 
 //! \brief Broadcasts the, non-dynamic, state from the master to all ranks in cr->mpi_comm_mygroup
 //
 // This is intended to be used with MPI parallelization without
 // domain decompostion (currently with NM and TPI).
-void broadcastStateWithoutDynamics(const t_commrec* cr, t_state* state);
+void broadcastStateWithoutDynamics(MPI_Comm communicator,
+                                   bool     useDomainDecomposition,
+                                   bool     isParallelRun,
+                                   t_state* state);
 
 //! \brief Broadcast inputrec and mtop and allocate node-specific settings
-void init_parallel(t_commrec*                  cr,
+void init_parallel(MPI_Comm                    communicator,
+                   bool                        isMasterRank,
                    t_inputrec*                 inputrec,
                    gmx_mtop_t*                 mtop,
                    PartialDeserializedTprFile* partialDeserializedTpr);

diff --git a/src/gromacs/mdlib/md_support.cpp b/src/gromacs/mdlib/md_support.cpp
index 33056bc855cf580e759f91c09bd65d88b06ab5c3..31df61f42ddc2b77efe05aac3c9920a5d95f69c8 100644 (file)
--- a/src/gromacs/mdlib/md_support.cpp
+++ b/src/gromacs/mdlib/md_support.cpp
@@ -497,7 +497,7 @@ void rerun_parallel_comm(t_commrec* cr, t_trxframe* fr, gmx_bool* bLastStep)
     }
     xp = fr->x;
     vp = fr->v;
-    gmx_bcast(sizeof(*fr), fr, cr);
+    gmx_bcast(sizeof(*fr), fr, cr->mpi_comm_mygroup);
     fr->x = xp;
     fr->v = vp;
 

diff --git a/src/gromacs/mdlib/simulationsignal.cpp b/src/gromacs/mdlib/simulationsignal.cpp
index 0f0a7f2ac5837ff5b1ca2f3dec494006d17386be..415c91b0b6a9bbecce1d6ca5cf9f24641b2ee3da 100644 (file)
--- a/src/gromacs/mdlib/simulationsignal.cpp
+++ b/src/gromacs/mdlib/simulationsignal.cpp
@@ -113,7 +113,7 @@ void SimulationSignaller::signalInterSim()
         gmx_sum_sim(eglsNR, mpiBuffer_.data(), ms_);
     }
     // Communicate the signals from the master to the others.
-    gmx_bcast(eglsNR * sizeof(mpiBuffer_[0]), mpiBuffer_.data(), cr_);
+    gmx_bcast(eglsNR * sizeof(mpiBuffer_[0]), mpiBuffer_.data(), cr_->mpi_comm_mygroup);
 }
 
 void SimulationSignaller::setSignals()

diff --git a/src/gromacs/mdlib/update.cpp b/src/gromacs/mdlib/update.cpp
index d9438d281f36727a894a086faf8aeb70a3b08f29..82d302c846f0614872a557d020c7f7f6c26f11b2 100644 (file)
--- a/src/gromacs/mdlib/update.cpp
+++ b/src/gromacs/mdlib/update.cpp
@@ -1312,22 +1312,27 @@ void restore_ekinstate_from_state(const t_commrec* cr, gmx_ekindata_t* ekind, co
 
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(n), &n, cr);
+        gmx_bcast(sizeof(n), &n, cr->mpi_comm_mygroup);
         for (i = 0; i < n; i++)
         {
-            gmx_bcast(DIM * DIM * sizeof(ekind->tcstat[i].ekinh[0][0]), ekind->tcstat[i].ekinh[0], cr);
-            gmx_bcast(DIM * DIM * sizeof(ekind->tcstat[i].ekinf[0][0]), ekind->tcstat[i].ekinf[0], cr);
+            gmx_bcast(DIM * DIM * sizeof(ekind->tcstat[i].ekinh[0][0]), ekind->tcstat[i].ekinh[0],
+                      cr->mpi_comm_mygroup);
+            gmx_bcast(DIM * DIM * sizeof(ekind->tcstat[i].ekinf[0][0]), ekind->tcstat[i].ekinf[0],
+                      cr->mpi_comm_mygroup);
             gmx_bcast(DIM * DIM * sizeof(ekind->tcstat[i].ekinh_old[0][0]),
-                      ekind->tcstat[i].ekinh_old[0], cr);
-
-            gmx_bcast(sizeof(ekind->tcstat[i].ekinscalef_nhc), &(ekind->tcstat[i].ekinscalef_nhc), cr);
-            gmx_bcast(sizeof(ekind->tcstat[i].ekinscaleh_nhc), &(ekind->tcstat[i].ekinscaleh_nhc), cr);
-            gmx_bcast(sizeof(ekind->tcstat[i].vscale_nhc), &(ekind->tcstat[i].vscale_nhc), cr);
+                      ekind->tcstat[i].ekinh_old[0], cr->mpi_comm_mygroup);
+
+            gmx_bcast(sizeof(ekind->tcstat[i].ekinscalef_nhc), &(ekind->tcstat[i].ekinscalef_nhc),
+                      cr->mpi_comm_mygroup);
+            gmx_bcast(sizeof(ekind->tcstat[i].ekinscaleh_nhc), &(ekind->tcstat[i].ekinscaleh_nhc),
+                      cr->mpi_comm_mygroup);
+            gmx_bcast(sizeof(ekind->tcstat[i].vscale_nhc), &(ekind->tcstat[i].vscale_nhc),
+                      cr->mpi_comm_mygroup);
         }
-        gmx_bcast(DIM * DIM * sizeof(ekind->ekin[0][0]), ekind->ekin[0], cr);
+        gmx_bcast(DIM * DIM * sizeof(ekind->ekin[0][0]), ekind->ekin[0], cr->mpi_comm_mygroup);
 
-        gmx_bcast(sizeof(ekind->dekindl), &ekind->dekindl, cr);
-        gmx_bcast(sizeof(ekind->cosacc.mvcos), &ekind->cosacc.mvcos, cr);
+        gmx_bcast(sizeof(ekind->dekindl), &ekind->dekindl, cr->mpi_comm_mygroup);
+        gmx_bcast(sizeof(ekind->cosacc.mvcos), &ekind->cosacc.mvcos, cr->mpi_comm_mygroup);
     }
 }
 

diff --git a/src/gromacs/mdrun/md.cpp b/src/gromacs/mdrun/md.cpp
index 91ec7978129e9e4a783a499b0da7d6c6fa840756..fa5b0750ebee08465df2055976b2aa35e1c180c7 100644 (file)
--- a/src/gromacs/mdrun/md.cpp
+++ b/src/gromacs/mdrun/md.cpp
@@ -546,7 +546,7 @@ void gmx::LegacySimulator::do_md()
     bool hasReadEkinState = MASTER(cr) ? state_global->ekinstate.hasReadEkinState : false;
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(hasReadEkinState), &hasReadEkinState, cr);
+        gmx_bcast(sizeof(hasReadEkinState), &hasReadEkinState, cr->mpi_comm_mygroup);
     }
     if (hasReadEkinState)
     {
@@ -737,7 +737,7 @@ void gmx::LegacySimulator::do_md()
                               "The initial step is not consistent across multi simulations which "
                               "share the state");
                 }
-                gmx_barrier(cr);
+                gmx_barrier(cr->mpi_comm_mygroup);
             }
             else
             {

diff --git a/src/gromacs/mdrun/mimic.cpp b/src/gromacs/mdrun/mimic.cpp
index a8130c680c8c81b22959e70be4e694ea94cf6d60..2cfa7705a0e94978fa03ec39b0471b2b419c6db8 100644 (file)
--- a/src/gromacs/mdrun/mimic.cpp
+++ b/src/gromacs/mdrun/mimic.cpp
@@ -262,7 +262,7 @@ void gmx::LegacySimulator::do_mimic()
                             imdSession, pull_work, state, &f, mdAtoms, &top, fr, vsite, constr,
                             nrnb, nullptr, FALSE);
         shouldCheckNumberOfBondedInteractions = true;
-        gmx_bcast(sizeof(ir->nsteps), &ir->nsteps, cr);
+        gmx_bcast(sizeof(ir->nsteps), &ir->nsteps, cr->mpi_comm_mygroup);
     }
     else
     {

diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp
index 8605f612edd9c48cbefdca2b21beb5cc8de4a6d0..55e34445723f0c3a989d22dbc6b80bf09ff69b82 100644 (file)
--- a/src/gromacs/mdrun/runner.cpp
+++ b/src/gromacs/mdrun/runner.cpp
@@ -843,7 +843,7 @@ int Mdrunner::mdrunner()
         {
             inputrec = &inputrecInstance;
         }
-        init_parallel(cr, inputrec, &mtop, partialDeserializedTpr.get());
+        init_parallel(cr->mpi_comm_mygroup, MASTER(cr), inputrec, &mtop, partialDeserializedTpr.get());
     }
     GMX_RELEASE_ASSERT(inputrec != nullptr, "All ranks should have a valid inputrec now");
     partialDeserializedTpr.reset(nullptr);
@@ -956,7 +956,7 @@ int Mdrunner::mdrunner()
         {
             globalState = std::make_unique<t_state>();
         }
-        broadcastStateWithoutDynamics(cr, globalState.get());
+        broadcastStateWithoutDynamics(cr->mpi_comm_mygroup, DOMAINDECOMP(cr), PAR(cr), globalState.get());
     }
 
     /* A parallel command line option consistency check that we can
@@ -1093,7 +1093,7 @@ int Mdrunner::mdrunner()
 
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(box), box, cr);
+        gmx_bcast(sizeof(box), box, cr->mpi_comm_mygroup);
     }
 
     if (inputrec->cutoff_scheme != ecutsVERLET)
@@ -1131,7 +1131,7 @@ int Mdrunner::mdrunner()
         }
     }
 
-    // Produce the task assignment for this rank.
+    // Produce the task assignment for this rank - done after DD is constructed
     GpuTaskAssignmentsBuilder gpuTaskAssignmentsBuilder;
     GpuTaskAssignments        gpuTaskAssignments = gpuTaskAssignmentsBuilder.build(
             gpuIdsToUse, userGpuTaskAssignment, *hwinfo, communicator, physicalNodeComm,
@@ -1328,7 +1328,7 @@ int Mdrunner::mdrunner()
         /* Master synchronizes its value of reset_counters with all nodes
          * including PME only nodes */
         int64_t reset_counters = wcycle_get_reset_counters(wcycle);
-        gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
+        gmx_bcast(sizeof(reset_counters), &reset_counters, cr->mpi_comm_mysim);
         wcycle_set_reset_counters(wcycle, reset_counters);
     }
 
@@ -1494,8 +1494,8 @@ int Mdrunner::mdrunner()
         if (cr->npmenodes > 0)
         {
             /* The PME only nodes need to know nChargePerturbed(FEP on Q) and nTypePerturbed(FEP on LJ)*/
-            gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
-            gmx_bcast_sim(sizeof(nTypePerturbed), &nTypePerturbed, cr);
+            gmx_bcast(sizeof(nChargePerturbed), &nChargePerturbed, cr->mpi_comm_mysim);
+            gmx_bcast(sizeof(nTypePerturbed), &nTypePerturbed, cr->mpi_comm_mysim);
         }
 
         if (thisRankHasDuty(cr, DUTY_PME))

diff --git a/src/gromacs/modularsimulator/modularsimulator.cpp b/src/gromacs/modularsimulator/modularsimulator.cpp
index da948918e76d4b2e788b17f80353610d084fa225..c71d731e5f293f36380b257eab936ba0c50c7113 100644 (file)
--- a/src/gromacs/modularsimulator/modularsimulator.cpp
+++ b/src/gromacs/modularsimulator/modularsimulator.cpp
@@ -350,7 +350,7 @@ void ModularSimulator::constructElementsAndSignallers()
     bool hasReadEkinState = MASTER(cr) ? state_global->ekinstate.hasReadEkinState : false;
     if (PAR(cr))
     {
-        gmx_bcast(sizeof(hasReadEkinState), &hasReadEkinState, cr);
+        gmx_bcast(sizeof(hasReadEkinState), &hasReadEkinState, cr->mpi_comm_mygroup);
     }
     if (hasReadEkinState)
     {

diff --git a/src/gromacs/pulling/pull.cpp b/src/gromacs/pulling/pull.cpp
index 712399a7f19f755f5de12f832bbcbe21df9ec7f0..314052e32a5ae08c44bce258905652bbfb6db505 100644 (file)
--- a/src/gromacs/pulling/pull.cpp
+++ b/src/gromacs/pulling/pull.cpp
@@ -1718,7 +1718,7 @@ void dd_make_local_pull_groups(const t_commrec* cr, struct pull_t* pull)
                                "date prev. COM "
                                "to bcast here as well as to e.g. checkpointing");
 
-                    gmx_bcast(sizeof(dvec), group.x_prev_step, cr);
+                    gmx_bcast(sizeof(dvec), group.x_prev_step, cr->mpi_comm_mygroup);
                 }
             }
         }
@@ -2314,7 +2314,8 @@ void preparePrevStepPullCom(const t_inputrec* ir,
         if (PAR(cr))
         {
             /* Only the master rank has the checkpointed COM from the previous step */
-            gmx_bcast(sizeof(double) * state->pull_com_prev_step.size(), &state->pull_com_prev_step[0], cr);
+            gmx_bcast(sizeof(double) * state->pull_com_prev_step.size(),
+                      &state->pull_com_prev_step[0], cr->mpi_comm_mygroup);
         }
         setPrevStepPullComFromState(pull_work, state);
     }

diff --git a/src/gromacs/pulling/pull_rotation.cpp b/src/gromacs/pulling/pull_rotation.cpp
index 641ffcf52877ee238f38f21d95079df78f770a56..9a95f2430d967e1fe7b5fc4a15f2ec07b2d82a44 100644 (file)
--- a/src/gromacs/pulling/pull_rotation.cpp
+++ b/src/gromacs/pulling/pull_rotation.cpp
@@ -3494,7 +3494,7 @@ static void init_rot_group(FILE*            fplog,
 #if GMX_MPI
         if (PAR(cr))
         {
-            gmx_bcast(sizeof(erg->xc_center), erg->xc_center, cr);
+            gmx_bcast(sizeof(erg->xc_center), erg->xc_center, cr->mpi_comm_mygroup);
         }
 #endif
     }
@@ -3529,7 +3529,7 @@ static void init_rot_group(FILE*            fplog,
 #if GMX_MPI
         if (PAR(cr))
         {
-            gmx_bcast(erg->rotg->nat * sizeof(erg->xc_old[0]), erg->xc_old, cr);
+            gmx_bcast(erg->rotg->nat * sizeof(erg->xc_old[0]), erg->xc_old, cr->mpi_comm_mygroup);
         }
 #endif
     }

diff --git a/src/gromacs/restraint/restraintmdmodule.cpp b/src/gromacs/restraint/restraintmdmodule.cpp
index cb8091aac40b45aed8b90926b3e15df26b4c34b8..d940a456675cb2801e1ceab3658d36bd3105529c 100644 (file)
--- a/src/gromacs/restraint/restraintmdmodule.cpp
+++ b/src/gromacs/restraint/restraintmdmodule.cpp
@@ -119,7 +119,7 @@ void RestraintForceProvider::calculateForces(const ForceProviderInput& forceProv
     {
         // Note: this assumes that all ranks are hitting this line, which is not generally true.
         // I need to find the right subcommunicator. What I really want is a _scoped_ communicator...
-        gmx_barrier(&cr);
+        gmx_barrier(cr.mpi_comm_mygroup);
     }
 
     // Apply restraint on all thread ranks only after any updates have been made.

diff --git a/src/gromacs/swap/swapcoords.cpp b/src/gromacs/swap/swapcoords.cpp
index 37df287b80a5d7e370bddf4d83065de8d51a2669..e6debca14eacd2f66ac809f715bf0b334961f9d0 100644 (file)
--- a/src/gromacs/swap/swapcoords.cpp
+++ b/src/gromacs/swap/swapcoords.cpp
@@ -933,9 +933,10 @@ static void bc_initial_concentrations(t_commrec* cr, t_swapcoords* swap, t_swap*
 
         for (ic = 0; ic < eCompNR; ic++)
         {
-            gmx_bcast(sizeof(g->comp[ic].nMolReq), &(g->comp[ic].nMolReq), cr);
-            gmx_bcast(sizeof(g->comp[ic].nMol), &(g->comp[ic].nMol), cr);
-            gmx_bcast(swap->nAverage * sizeof(g->comp[ic].nMolPast[0]), g->comp[ic].nMolPast, cr);
+            gmx_bcast(sizeof(g->comp[ic].nMolReq), &(g->comp[ic].nMolReq), cr->mpi_comm_mygroup);
+            gmx_bcast(sizeof(g->comp[ic].nMol), &(g->comp[ic].nMol), cr->mpi_comm_mygroup);
+            gmx_bcast(swap->nAverage * sizeof(g->comp[ic].nMolPast[0]), g->comp[ic].nMolPast,
+                      cr->mpi_comm_mygroup);
         }
     }
 }
@@ -1562,7 +1563,8 @@ t_swap* init_swapcoords(FILE*                       fplog,
         for (int ig = eGrpSplit0; ig <= eGrpSplit1; ig++)
         {
             g = &(s->group[ig]);
-            gmx_bcast((g->atomset.numAtomsGlobal()) * sizeof((g->xc_old)[0]), g->xc_old, (cr));
+            gmx_bcast((g->atomset.numAtomsGlobal()) * sizeof((g->xc_old)[0]), g->xc_old,
+                      cr->mpi_comm_mygroup);
         }
     }
 

diff --git a/src/gromacs/tools/pme_error.cpp b/src/gromacs/tools/pme_error.cpp
index 5b503f30d78b65e35be87c07ec07ce78e7ae7edd..cb8087f6e707d59b18977753df418e07eba35080 100644 (file)
--- a/src/gromacs/tools/pme_error.cpp
+++ b/src/gromacs/tools/pme_error.cpp
@@ -647,7 +647,7 @@ static real estimate_reciprocal(t_inputinfo* info,
         /* Broadcast the random number array to the other nodes */
         if (PAR(cr))
         {
-            nblock_bc(cr, xtot, numbers);
+            nblock_bc(cr->mpi_comm_mygroup, xtot, numbers);
         }
 
         if (bVerbose && MASTER(cr))
@@ -825,11 +825,11 @@ static int prepare_x_q(real* q[], rvec* x[], const gmx_mtop_t* mtop, const rvec
     if (PAR(cr))
     {
         /* Transfer the number of charges */
-        block_bc(cr, nq);
-        snew_bc(cr, *x, nq);
-        snew_bc(cr, *q, nq);
-        nblock_bc(cr, nq, *x);
-        nblock_bc(cr, nq, *q);
+        block_bc(cr->mpi_comm_mygroup, nq);
+        snew_bc(MASTER(cr), *x, nq);
+        snew_bc(MASTER(cr), *q, nq);
+        nblock_bc(cr->mpi_comm_mygroup, nq, *x);
+        nblock_bc(cr->mpi_comm_mygroup, nq, *q);
     }
 
     return nq;
@@ -884,20 +884,20 @@ static void read_tpr_file(const char*  fn_sim_tpr,
 /* Transfer what we need for parallelizing the reciprocal error estimate */
 static void bcast_info(t_inputinfo* info, const t_commrec* cr)
 {
-    nblock_bc(cr, info->n_entries, info->nkx);
-    nblock_bc(cr, info->n_entries, info->nky);
-    nblock_bc(cr, info->n_entries, info->nkz);
-    nblock_bc(cr, info->n_entries, info->ewald_beta);
-    nblock_bc(cr, info->n_entries, info->pme_order);
-    nblock_bc(cr, info->n_entries, info->e_dir);
-    nblock_bc(cr, info->n_entries, info->e_rec);
-    block_bc(cr, info->volume);
-    block_bc(cr, info->recipbox);
-    block_bc(cr, info->natoms);
-    block_bc(cr, info->fracself);
-    block_bc(cr, info->bTUNE);
-    block_bc(cr, info->q2all);
-    block_bc(cr, info->q2allnr);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->nkx);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->nky);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->nkz);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->ewald_beta);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->pme_order);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->e_dir);
+    nblock_bc(cr->mpi_comm_mygroup, info->n_entries, info->e_rec);
+    block_bc(cr->mpi_comm_mygroup, info->volume);
+    block_bc(cr->mpi_comm_mygroup, info->recipbox);
+    block_bc(cr->mpi_comm_mygroup, info->natoms);
+    block_bc(cr->mpi_comm_mygroup, info->fracself);
+    block_bc(cr->mpi_comm_mygroup, info->bTUNE);
+    block_bc(cr->mpi_comm_mygroup, info->q2all);
+    block_bc(cr->mpi_comm_mygroup, info->q2allnr);
 }