From: Berk Hess Date: Wed, 18 Aug 2021 06:44:03 +0000 (+0000) Subject: Fix mdrun -ddorder pp_pme X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=81327ca7bdff3e7aba2a07a01e7a6123f6876c17;p=alexxy%2Fgromacs.git Fix mdrun -ddorder pp_pme --- diff --git a/docs/release-notes/2021/2021.3.rst b/docs/release-notes/2021/2021.3.rst index fc8e40e769..b9ff941e91 100644 --- a/docs/release-notes/2021/2021.3.rst +++ b/docs/release-notes/2021/2021.3.rst @@ -16,6 +16,14 @@ in the :ref:`release-notes`. Fixes where mdrun could behave incorrectly ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Fix mdrun -ddorder pp_pme +""""""""""""""""""""""""" + +When rank ordering PP-PME, mdrun would deadlock during the initialization +phase. + +:issue:`4114` + Fixed gmxapi MD plugin binding """""""""""""""""""""""""""""" diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index d3ba5aba08..8e27798962 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -4,7 +4,7 @@ * Copyright (c) 2005,2006,2007,2008,2009 by the GROMACS development team. * Copyright (c) 2010,2011,2012,2013,2014 by the GROMACS development team. * Copyright (c) 2015,2016,2017,2018,2019 by the GROMACS development team. - * Copyright (c) 2020, by the GROMACS development team, led by + * Copyright (c) 2020,2021, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -721,7 +721,8 @@ static int ddcoord2simnodeid(const t_commrec* cr, int x, int y, int z) } else { - if (cr->dd->comm->ddRankSetup.usePmeOnlyRanks) + const DDRankSetup& rankSetup = cr->dd->comm->ddRankSetup; + if (rankSetup.rankOrder != DdRankOrder::pp_pme && rankSetup.usePmeOnlyRanks) { nodeid = ddindex + gmx_ddcoord2pmeindex(cr, x, y, z); } @@ -1448,7 +1449,6 @@ static void receive_ddindex2simnodeid(gmx_domdec_t* dd, t_commrec* cr) static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog, t_commrec* cr, - const DdRankOrder ddRankOrder, bool gmx_unused reorder, const DDRankSetup& ddRankSetup, ivec ddCellIndex, @@ -1456,6 +1456,8 @@ static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog, { CartesianRankSetup cartSetup; + const DdRankOrder ddRankOrder = ddRankSetup.rankOrder; + cartSetup.bCartesianPP = (ddRankOrder == DdRankOrder::cartesian); cartSetup.bCartesianPP_PME = false; @@ -1605,7 +1607,6 @@ static CartesianRankSetup split_communicator(const gmx::MDLogger& mdlog, */ static CartesianRankSetup makeGroupCommunicators(const gmx::MDLogger& mdlog, const DDSettings& ddSettings, - const DdRankOrder ddRankOrder, const DDRankSetup& ddRankSetup, t_commrec* cr, ivec ddCellIndex, @@ -1623,8 +1624,8 @@ static CartesianRankSetup makeGroupCommunicators(const gmx::MDLogger& mdlog, if (ddRankSetup.usePmeOnlyRanks) { /* Split the communicator into a PP and PME part */ - cartSetup = split_communicator(mdlog, cr, ddRankOrder, ddSettings.useCartesianReorder, - ddRankSetup, ddCellIndex, pmeRanks); + cartSetup = split_communicator(mdlog, cr, ddSettings.useCartesianReorder, ddRankSetup, + ddCellIndex, pmeRanks); } else { @@ -2307,6 +2308,7 @@ static void checkDDGridSetup(const DDGridSetup& ddGridSetup, /*! \brief Set the cell size and interaction limits, as well as the DD grid */ static DDRankSetup getDDRankSetup(const gmx::MDLogger& mdlog, int numNodes, + const DdRankOrder rankOrder, const DDGridSetup& ddGridSetup, const t_inputrec& ir) { @@ -2317,6 +2319,8 @@ static DDRankSetup getDDRankSetup(const gmx::MDLogger& mdlog, DDRankSetup ddRankSetup; + ddRankSetup.rankOrder = rankOrder; + ddRankSetup.numPPRanks = numNodes - ddGridSetup.numPmeOnlyRanks; copy_ivec(ddGridSetup.numDomains, ddRankSetup.numPPCells); @@ -3008,11 +3012,11 @@ DomainDecompositionBuilder::Impl::Impl(const MDLogger& mdlog, cr_->npmenodes = ddGridSetup_.numPmeOnlyRanks; - ddRankSetup_ = getDDRankSetup(mdlog_, cr_->sizeOfDefaultCommunicator, ddGridSetup_, ir_); + ddRankSetup_ = getDDRankSetup(mdlog_, cr_->sizeOfDefaultCommunicator, options_.rankOrder, + ddGridSetup_, ir_); /* Generate the group communicator, also decides the duty of each rank */ - cartSetup_ = makeGroupCommunicators(mdlog_, ddSettings_, options_.rankOrder, ddRankSetup_, cr_, - ddCellIndex_, &pmeRanks_); + cartSetup_ = makeGroupCommunicators(mdlog_, ddSettings_, ddRankSetup_, cr_, ddCellIndex_, &pmeRanks_); } gmx_domdec_t* DomainDecompositionBuilder::Impl::build(LocalAtomSetManager* atomSets) diff --git a/src/gromacs/domdec/domdec_internal.h b/src/gromacs/domdec/domdec_internal.h index 691b0b87aa..f2aedaec7a 100644 --- a/src/gromacs/domdec/domdec_internal.h +++ b/src/gromacs/domdec/domdec_internal.h @@ -2,7 +2,7 @@ * This file is part of the GROMACS molecular simulation package. * * Copyright (c) 2014,2015,2016,2017,2018 by the GROMACS development team. - * Copyright (c) 2019,2020, by the GROMACS development team, led by + * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -60,6 +60,13 @@ struct t_commrec; struct BalanceRegion; +namespace gmx +{ +enum class DdRankOrder : int; +} +// namespace + + //! Indices to communicate in a dimension struct gmx_domdec_ind_t { @@ -497,6 +504,9 @@ struct DDSettings /*! \brief Information on how the DD ranks are set up */ struct DDRankSetup { + /**< The rank ordering */ + gmx::DdRankOrder rankOrder; + /**< The number of particle-particle (non PME-only) ranks */ int numPPRanks = 0; /**< The DD PP grid */