Fix mdrun -ddorder pp_pme

diff --git a/docs/release-notes/2021/2021.3.rst b/docs/release-notes/2021/2021.3.rst
index fc8e40e7692d76d1d28281120ef2423fb45ce0cb..b9ff941e9113f6073935db0d9e51e0156ba4a75a 100644 (file)
--- a/docs/release-notes/2021/2021.3.rst
+++ b/docs/release-notes/2021/2021.3.rst
@@ -16,6 +16,14 @@ in the :ref:`release-notes`.
 Fixes where mdrun could behave incorrectly
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+Fix mdrun -ddorder pp_pme
+"""""""""""""""""""""""""
+
+When rank ordering PP-PME, mdrun would deadlock during the initialization
+phase.
+
+:issue:`4114`
+
 Fixed gmxapi MD plugin binding
 """"""""""""""""""""""""""""""
 

diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp
index d3ba5aba08ef2f8e997bb988ab5baab865303824..8e27798962354c15ebac3c701defa5ce86da08fd 100644 (file)
--- a/src/gromacs/domdec/domdec.cpp
+++ b/src/gromacs/domdec/domdec.cpp
@@ -4,7 +4,7 @@
  * Copyright (c) 2005,2006,2007,2008,2009 by the GROMACS development team.
  * Copyright (c) 2010,2011,2012,2013,2014 by the GROMACS development team.
  * Copyright (c) 2015,2016,2017,2018,2019 by the GROMACS development team.
- * Copyright (c) 2020, by the GROMACS development team, led by
+ * Copyright (c) 2020,2021, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -721,7 +721,8 @@ static int ddcoord2simnodeid(const t_commrec* cr, int x, int y, int z)
         }
         else
         {
-            if (cr->dd->comm->ddRankSetup.usePmeOnlyRanks)
+            const DDRankSetup& rankSetup = cr->dd->comm->ddRankSetup;
+            if (rankSetup.rankOrder != DdRankOrder::pp_pme && rankSetup.usePmeOnlyRanks)
             {
                 nodeid = ddindex + gmx_ddcoord2pmeindex(cr, x, y, z);
             }
@@ -1448,7 +1449,6 @@ static void receive_ddindex2simnodeid(gmx_domdec_t* dd, t_commrec* cr)
 
 static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog,
                                              t_commrec*           cr,
-                                             const DdRankOrder    ddRankOrder,
                                              bool gmx_unused    reorder,
                                              const DDRankSetup& ddRankSetup,
                                              ivec               ddCellIndex,
@@ -1456,6 +1456,8 @@ static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog,
 {
     CartesianRankSetup cartSetup;
 
+    const DdRankOrder ddRankOrder = ddRankSetup.rankOrder;
+
     cartSetup.bCartesianPP     = (ddRankOrder == DdRankOrder::cartesian);
     cartSetup.bCartesianPP_PME = false;
 
@@ -1605,7 +1607,6 @@ static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog,
  */
 static CartesianRankSetup makeGroupCommunicators(const gmx::MDLogger& mdlog,
                                                  const DDSettings&    ddSettings,
-                                                 const DdRankOrder    ddRankOrder,
                                                  const DDRankSetup&   ddRankSetup,
                                                  t_commrec*           cr,
                                                  ivec                 ddCellIndex,
@@ -1623,8 +1624,8 @@ static CartesianRankSetup makeGroupCommunicators(const gmx::MDLogger& mdlog,
     if (ddRankSetup.usePmeOnlyRanks)
     {
         /* Split the communicator into a PP and PME part */
-        cartSetup = split_communicator(mdlog, cr, ddRankOrder, ddSettings.useCartesianReorder,
-                                       ddRankSetup, ddCellIndex, pmeRanks);
+        cartSetup = split_communicator(mdlog, cr, ddSettings.useCartesianReorder, ddRankSetup,
+                                       ddCellIndex, pmeRanks);
     }
     else
     {
@@ -2307,6 +2308,7 @@ static void checkDDGridSetup(const DDGridSetup&   ddGridSetup,
 /*! \brief Set the cell size and interaction limits, as well as the DD grid */
 static DDRankSetup getDDRankSetup(const gmx::MDLogger& mdlog,
                                   int                  numNodes,
+                                  const DdRankOrder    rankOrder,
                                   const DDGridSetup&   ddGridSetup,
                                   const t_inputrec&    ir)
 {
@@ -2317,6 +2319,8 @@ static DDRankSetup getDDRankSetup(const gmx::MDLogger& mdlog,
 
     DDRankSetup ddRankSetup;
 
+    ddRankSetup.rankOrder = rankOrder;
+
     ddRankSetup.numPPRanks = numNodes - ddGridSetup.numPmeOnlyRanks;
     copy_ivec(ddGridSetup.numDomains, ddRankSetup.numPPCells);
 
@@ -3008,11 +3012,11 @@ DomainDecompositionBuilder::Impl::Impl(const MDLogger&      mdlog,
 
     cr_->npmenodes = ddGridSetup_.numPmeOnlyRanks;
 
-    ddRankSetup_ = getDDRankSetup(mdlog_, cr_->sizeOfDefaultCommunicator, ddGridSetup_, ir_);
+    ddRankSetup_ = getDDRankSetup(mdlog_, cr_->sizeOfDefaultCommunicator, options_.rankOrder,
+                                  ddGridSetup_, ir_);
 
     /* Generate the group communicator, also decides the duty of each rank */
-    cartSetup_ = makeGroupCommunicators(mdlog_, ddSettings_, options_.rankOrder, ddRankSetup_, cr_,
-                                        ddCellIndex_, &pmeRanks_);
+    cartSetup_ = makeGroupCommunicators(mdlog_, ddSettings_, ddRankSetup_, cr_, ddCellIndex_, &pmeRanks_);
 }
 
 gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets)

diff --git a/src/gromacs/domdec/domdec_internal.h b/src/gromacs/domdec/domdec_internal.h
index 691b0b87aa4cef053579f538791b82c868beb2d7..f2aedaec7a5d1c8a08cc27191e2fa8d243a7f0ac 100644 (file)
--- a/src/gromacs/domdec/domdec_internal.h
+++ b/src/gromacs/domdec/domdec_internal.h
@@ -2,7 +2,7 @@
  * This file is part of the GROMACS molecular simulation package.
  *
  * Copyright (c) 2014,2015,2016,2017,2018 by the GROMACS development team.
- * Copyright (c) 2019,2020, by the GROMACS development team, led by
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -60,6 +60,13 @@ struct t_commrec;
 
 struct BalanceRegion;
 
+namespace gmx
+{
+enum class DdRankOrder : int;
+}
+// namespace
+
+
 //! Indices to communicate in a dimension
 struct gmx_domdec_ind_t
 {
@@ -497,6 +504,9 @@ struct DDSettings
 /*! \brief Information on how the DD ranks are set up */
 struct DDRankSetup
 {
+    /**< The rank ordering */
+    gmx::DdRankOrder rankOrder;
+
     /**< The number of particle-particle (non PME-only) ranks */
     int numPPRanks = 0;
     /**< The DD PP grid */