From e0f481ae943b4fac5ddd3c6ed7f1af981f1d7de6 Mon Sep 17 00:00:00 2001 From: Mark Abraham Date: Wed, 15 Jul 2020 17:09:31 +0000 Subject: [PATCH] Divide default communicator from DD communicators The communicators mpi_comm_mysim and mpi_comm_mygroup inside t_commrec got initialized in init_commrec (to MPI_COMM_WORLD if no multisim, to a subset otherwise). These communicators were then used in subsequent setup work, before they got reassigned during the construction of the DDBuilder object and the construction of the actual domain decomposition object. Effectively, this means that the same communicators (and, hence, identical function calls) do very different things depending on whether they get used before or after the setup of domain decomposition. It also means that before DD set up, mpi_comm_mysim and mpi_comm_mygroup are *identical*. This change introduces an additional communicator within t_commrec, mpiDefaulCommunicator, which helps to make these implicit assumptions explicit. Consequently, this also redefines PAR(cr), MASTER(cr), and SIMMASTER(cr). This change will allow to move the sim and group communicators, which are now only created at DD time, into the DD object, logically separating the DD object from t_commrec. Refs #2395 --- .../applied_forces/tests/electricfield.cpp | 2 +- src/gromacs/awh/awh.cpp | 2 +- src/gromacs/awh/biassharing.cpp | 12 +- src/gromacs/awh/biasstate.cpp | 8 +- src/gromacs/gmxlib/network.cpp | 38 +--- src/gromacs/gmxlib/network.h | 3 +- src/gromacs/listed_forces/disre.cpp | 8 +- src/gromacs/listed_forces/orires.cpp | 5 +- src/gromacs/mdlib/lincs.cpp | 2 +- src/gromacs/mdlib/md_support.cpp | 20 +- src/gromacs/mdlib/mdoutf.cpp | 6 +- src/gromacs/mdlib/tests/constrtestrunners.cpp | 2 +- src/gromacs/mdrun/replicaexchange.cpp | 35 ++-- src/gromacs/mdrun/runner.cpp | 7 +- src/gromacs/mdrun/simulationcontext.cpp | 6 +- src/gromacs/mdrunutility/handlerestart.cpp | 4 +- src/gromacs/mdrunutility/multisim.cpp | 193 +++++++++--------- src/gromacs/mdrunutility/multisim.h | 74 +++++-- .../taskassignment/resourcedivision.cpp | 2 +- src/gromacs/tools/pme_error.cpp | 2 +- src/gromacs/utility/futil.cpp | 4 +- src/gromacs/utility/mpiinplacebuffers.cpp | 58 ------ src/gromacs/utility/mpiinplacebuffers.h | 71 ------- 23 files changed, 223 insertions(+), 341 deletions(-) delete mode 100644 src/gromacs/utility/mpiinplacebuffers.cpp delete mode 100644 src/gromacs/utility/mpiinplacebuffers.h diff --git a/src/gromacs/applied_forces/tests/electricfield.cpp b/src/gromacs/applied_forces/tests/electricfield.cpp index 62c5b3199f..cf7ed342ee 100644 --- a/src/gromacs/applied_forces/tests/electricfield.cpp +++ b/src/gromacs/applied_forces/tests/electricfield.cpp @@ -109,7 +109,7 @@ public: std::vector chargeA{ 1 }; md.homenr = ssize(chargeA); md.chargeA = chargeA.data(); - CommrecHandle cr = init_commrec(MPI_COMM_WORLD, nullptr); + CommrecHandle cr = init_commrec(MPI_COMM_WORLD); matrix boxDummy = { { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 } }; ForceProviderInput forceProviderInput({}, md, 0.0, boxDummy, *cr); diff --git a/src/gromacs/awh/awh.cpp b/src/gromacs/awh/awh.cpp index 26a1a6a52d..65c9837b4d 100644 --- a/src/gromacs/awh/awh.cpp +++ b/src/gromacs/awh/awh.cpp @@ -149,7 +149,7 @@ Awh::Awh(FILE* fplog, int numSharingSimulations = 1; if (awhParams.shareBiasMultisim && isMultiSim(multiSimRecord_)) { - numSharingSimulations = multiSimRecord_->nsim; + numSharingSimulations = multiSimRecord_->numSimulations_; } /* Initialize all the biases */ diff --git a/src/gromacs/awh/biassharing.cpp b/src/gromacs/awh/biassharing.cpp index 52d5a89b4e..b377931fa0 100644 --- a/src/gromacs/awh/biassharing.cpp +++ b/src/gromacs/awh/biassharing.cpp @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -83,7 +83,7 @@ void biasesAreCompatibleForSharingBetweenSimulations(const AwhParams& const std::vector& pointSize, const gmx_multisim_t* multiSimComm) { - const int numSim = multiSimComm->nsim; + const int numSim = multiSimComm->numSimulations_; /* We currently enforce subsequent shared biases to have consecutive * share-group values starting at 1. This means we can reduce shared @@ -105,7 +105,7 @@ void biasesAreCompatibleForSharingBetweenSimulations(const AwhParams& } } std::vector numShareAll(numSim); - numShareAll[multiSimComm->sim] = numShare; + numShareAll[multiSimComm->simulationIndex_] = numShare; gmx_sumi_sim(numShareAll.size(), numShareAll.data(), multiSimComm); for (int sim = 1; sim < numSim; sim++) { @@ -117,8 +117,8 @@ void biasesAreCompatibleForSharingBetweenSimulations(const AwhParams& } std::vector intervals(numSim * 2); - intervals[numSim * 0 + multiSimComm->sim] = awhParams.nstSampleCoord; - intervals[numSim * 1 + multiSimComm->sim] = awhParams.numSamplesUpdateFreeEnergy; + intervals[numSim * 0 + multiSimComm->simulationIndex_] = awhParams.nstSampleCoord; + intervals[numSim * 1 + multiSimComm->simulationIndex_] = awhParams.numSamplesUpdateFreeEnergy; gmx_sumi_sim(intervals.size(), intervals.data(), multiSimComm); for (int sim = 1; sim < numSim; sim++) { @@ -142,7 +142,7 @@ void biasesAreCompatibleForSharingBetweenSimulations(const AwhParams& if (awhParams.awhBiasParams[b].shareGroup > 0) { std::vector pointSizes(numSim); - pointSizes[multiSimComm->sim] = pointSize[b]; + pointSizes[multiSimComm->simulationIndex_] = pointSize[b]; gmx_sumli_sim(pointSizes.size(), pointSizes.data(), multiSimComm); for (int sim = 1; sim < numSim; sim++) { diff --git a/src/gromacs/awh/biasstate.cpp b/src/gromacs/awh/biasstate.cpp index 8a90b36bc5..c63749e889 100644 --- a/src/gromacs/awh/biasstate.cpp +++ b/src/gromacs/awh/biasstate.cpp @@ -153,9 +153,9 @@ void sumPmf(gmx::ArrayRef pointState, { return; } - GMX_ASSERT(multiSimComm != nullptr && numSharedUpdate % multiSimComm->nsim == 0, - "numSharedUpdate should be a multiple of multiSimComm->nsim"); - GMX_ASSERT(numSharedUpdate == multiSimComm->nsim, + GMX_ASSERT(multiSimComm != nullptr && numSharedUpdate % multiSimComm->numSimulations_ == 0, + "numSharedUpdate should be a multiple of multiSimComm->numSimulations_"); + GMX_ASSERT(numSharedUpdate == multiSimComm->numSimulations_, "Sharing within a simulation is not implemented (yet)"); std::vector buffer(pointState.size()); @@ -734,7 +734,7 @@ void sumHistograms(gmx::ArrayRef pointState, /* Sum histograms over multiple simulations if needed. */ if (numSharedUpdate > 1) { - GMX_ASSERT(numSharedUpdate == multiSimComm->nsim, + GMX_ASSERT(numSharedUpdate == multiSimComm->numSimulations_, "Sharing within a simulation is not implemented (yet)"); /* Collect the weights and counts in linear arrays to be able to use gmx_sumd_sim. */ diff --git a/src/gromacs/gmxlib/network.cpp b/src/gromacs/gmxlib/network.cpp index 8b50570a22..66023edac8 100644 --- a/src/gromacs/gmxlib/network.cpp +++ b/src/gromacs/gmxlib/network.cpp @@ -47,7 +47,6 @@ #include #include "gromacs/commandline/filenm.h" -#include "gromacs/mdrunutility/multisim.h" #include "gromacs/mdtypes/commrec.h" #include "gromacs/utility/basenetwork.h" #include "gromacs/utility/cstringutil.h" @@ -55,14 +54,13 @@ #include "gromacs/utility/futil.h" #include "gromacs/utility/gmxassert.h" #include "gromacs/utility/gmxmpi.h" -#include "gromacs/utility/mpiinplacebuffers.h" #include "gromacs/utility/real.h" #include "gromacs/utility/smalloc.h" /* The source code in this file should be thread-safe. Please keep it that way. */ -CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms) +CommrecHandle init_commrec(MPI_Comm communicator) { CommrecHandle handle; t_commrec* cr; @@ -83,22 +81,10 @@ CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms) sizeOfCommunicator = 1; #endif - if (ms != nullptr) - { -#if GMX_MPI - MPI_Comm_split(communicator, ms->sim, rankInCommunicator, &cr->mpiDefaultCommunicator); - cr->sizeOfDefaultCommunicator = sizeOfCommunicator / ms->nsim; - MPI_Comm_rank(cr->mpiDefaultCommunicator, &cr->rankInDefaultCommunicator); -#else - gmx_fatal(FARGS, "Multisim can only run with MPI."); -#endif - } - else - { - cr->mpiDefaultCommunicator = communicator; - cr->sizeOfDefaultCommunicator = sizeOfCommunicator; - cr->rankInDefaultCommunicator = rankInCommunicator; - } + cr->mpiDefaultCommunicator = communicator; + cr->sizeOfDefaultCommunicator = sizeOfCommunicator; + cr->rankInDefaultCommunicator = rankInCommunicator; + // For now, we want things to go horribly wrong if this is used too early... // TODO: Remove when communicators are removed from commrec (#2395) cr->nnodes = -1; @@ -110,19 +96,6 @@ CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms) // TODO cr->duty should not be initialized here cr->duty = (DUTY_PP | DUTY_PME); -#if GMX_MPI && !MPI_IN_PLACE_EXISTS - /* initialize the MPI_IN_PLACE replacement buffers */ - snew(cr->mpb, 1); - cr->mpb->ibuf = nullptr; - cr->mpb->libuf = nullptr; - cr->mpb->fbuf = nullptr; - cr->mpb->dbuf = nullptr; - cr->mpb->ibuf_alloc = 0; - cr->mpb->libuf_alloc = 0; - cr->mpb->fbuf_alloc = 0; - cr->mpb->dbuf_alloc = 0; -#endif - return handle; } @@ -135,7 +108,6 @@ void done_commrec(t_commrec* cr) // TODO: implement // done_domdec(cr->dd); } - done_mpi_in_place_buf(cr->mpb); } #if GMX_MPI // TODO We need to be able to free communicators, but the diff --git a/src/gromacs/gmxlib/network.h b/src/gromacs/gmxlib/network.h index c17289a4ce..3bc41df9b3 100644 --- a/src/gromacs/gmxlib/network.h +++ b/src/gromacs/gmxlib/network.h @@ -49,7 +49,6 @@ #include "gromacs/utility/stringutil.h" #include "gromacs/utility/unique_cptr.h" -struct gmx_multisim_t; struct t_commrec; struct t_filenm; @@ -60,7 +59,7 @@ void done_commrec(t_commrec* cr); using CommrecHandle = gmx::unique_cptr; //! Allocate, initialize and return the commrec. -CommrecHandle init_commrec(MPI_Comm communicator, const gmx_multisim_t* ms); +CommrecHandle init_commrec(MPI_Comm communicator); struct t_commrec* reinitialize_commrec_for_this_thread(const t_commrec* cro); diff --git a/src/gromacs/listed_forces/disre.cpp b/src/gromacs/listed_forces/disre.cpp index 1c38611eb3..d95867cf7d 100644 --- a/src/gromacs/listed_forces/disre.cpp +++ b/src/gromacs/listed_forces/disre.cpp @@ -237,22 +237,22 @@ void init_disres(FILE* fplog, gmx_bcast(sizeof(int), &dd->nsystems, communicator); /* We use to allow any value of nsystems which was a divisor - * of ms->nsim. But this required an extra communicator which + * of ms->numSimulations_. But this required an extra communicator which * pulled in mpi.h in nearly all C files. */ - if (!(ms->nsim == 1 || ms->nsim == dd->nsystems)) + if (!(ms->numSimulations_ == 1 || ms->numSimulations_ == dd->nsystems)) { gmx_fatal(FARGS, "GMX_DISRE_ENSEMBLE_SIZE (%d) is not equal to 1 or the number of systems " "(option -multidir) %d", - dd->nsystems, ms->nsim); + dd->nsystems, ms->numSimulations_); } if (fplog) { fprintf(fplog, "Our ensemble consists of systems:"); for (int i = 0; i < dd->nsystems; i++) { - fprintf(fplog, " %d", (ms->sim / dd->nsystems) * dd->nsystems + i); + fprintf(fplog, " %d", (ms->simulationIndex_ / dd->nsystems) * dd->nsystems + i); } fprintf(fplog, "\n"); } diff --git a/src/gromacs/listed_forces/orires.cpp b/src/gromacs/listed_forces/orires.cpp index 4b35af3192..aab79ed972 100644 --- a/src/gromacs/listed_forces/orires.cpp +++ b/src/gromacs/listed_forces/orires.cpp @@ -282,7 +282,8 @@ void init_orires(FILE* fplog, if (ms) { - fprintf(fplog, " the orientation restraints are ensemble averaged over %d systems\n", ms->nsim); + fprintf(fplog, " the orientation restraints are ensemble averaged over %d systems\n", + ms->numSimulations_); check_multi_int(fplog, ms, od->nr, "the number of orientation restraints", FALSE); check_multi_int(fplog, ms, od->nref, "the number of fit atoms for orientation restraining", FALSE); @@ -432,7 +433,7 @@ real calc_orires_dev(const gmx_multisim_t* ms, if (ms) { - invn = 1.0 / ms->nsim; + invn = 1.0 / ms->numSimulations_; } else { diff --git a/src/gromacs/mdlib/lincs.cpp b/src/gromacs/mdlib/lincs.cpp index b9fff7fe05..10d8acf46b 100644 --- a/src/gromacs/mdlib/lincs.cpp +++ b/src/gromacs/mdlib/lincs.cpp @@ -2410,7 +2410,7 @@ bool constrain_lincs(bool computeRmsd, std::string simMesg; if (isMultiSim(ms)) { - simMesg += gmx::formatString(" in simulation %d", ms->sim); + simMesg += gmx::formatString(" in simulation %d", ms->simulationIndex_); } fprintf(stderr, "\nStep %" PRId64 diff --git a/src/gromacs/mdlib/md_support.cpp b/src/gromacs/mdlib/md_support.cpp index efad98cc8a..9b04614c8b 100644 --- a/src/gromacs/mdlib/md_support.cpp +++ b/src/gromacs/mdlib/md_support.cpp @@ -88,12 +88,12 @@ bool multisim_int_all_are_equal(const gmx_multisim_t* ms, int64_t value) GMX_RELEASE_ASSERT(ms, "Invalid use of multi-simulation pointer"); - snew(buf, ms->nsim); + snew(buf, ms->numSimulations_); /* send our value to all other master ranks, receive all of theirs */ - buf[ms->sim] = value; - gmx_sumli_sim(ms->nsim, buf, ms); + buf[ms->simulationIndex_] = value; + gmx_sumli_sim(ms->numSimulations_, buf, ms); - for (int s = 0; s < ms->nsim; s++) + for (int s = 0; s < ms->numSimulations_; s++) { if (buf[s] != value) { @@ -113,12 +113,12 @@ int multisim_min(const gmx_multisim_t* ms, int nmin, int n) gmx_bool bPos, bEqual; int s, d; - snew(buf, ms->nsim); - buf[ms->sim] = n; - gmx_sumi_sim(ms->nsim, buf, ms); + snew(buf, ms->numSimulations_); + buf[ms->simulationIndex_] = n; + gmx_sumi_sim(ms->numSimulations_, buf, ms); bPos = TRUE; bEqual = TRUE; - for (s = 0; s < ms->nsim; s++) + for (s = 0; s < ms->numSimulations_; s++) { bPos = bPos && (buf[s] > 0); bEqual = bEqual && (buf[s] == buf[0]); @@ -135,11 +135,11 @@ int multisim_min(const gmx_multisim_t* ms, int nmin, int n) for (d = 2; d < nmin; d++) { s = 0; - while (s < ms->nsim && d % buf[s] == 0) + while (s < ms->numSimulations_ && d % buf[s] == 0) { s++; } - if (s == ms->nsim) + if (s == ms->numSimulations_) { /* We found the LCM and it is less than nmin */ nmin = d; diff --git a/src/gromacs/mdlib/mdoutf.cpp b/src/gromacs/mdlib/mdoutf.cpp index 92874bb8f4..15c6517fc4 100644 --- a/src/gromacs/mdlib/mdoutf.cpp +++ b/src/gromacs/mdlib/mdoutf.cpp @@ -85,7 +85,7 @@ struct gmx_mdoutf gmx::IMDOutputProvider* outputProvider; const gmx::MdModulesNotifier* mdModulesNotifier; bool simulationsShareState; - MPI_Comm mpiCommMasters; + MPI_Comm mastersComm; }; @@ -133,7 +133,7 @@ gmx_mdoutf_t init_mdoutf(FILE* fplog, of->simulationsShareState = simulationsShareState; if (of->simulationsShareState) { - of->mpiCommMasters = ms->mpi_comm_masters; + of->mastersComm = ms->mastersComm_; } if (MASTER(cr)) @@ -320,7 +320,7 @@ void mdoutf_write_to_trajectory_files(FILE* fplog, DOMAINDECOMP(cr) ? cr->dd->nnodes : cr->nnodes, of->eIntegrator, of->simulation_part, of->bExpanded, of->elamstats, step, t, state_global, observablesHistory, *(of->mdModulesNotifier), - of->simulationsShareState, of->mpiCommMasters); + of->simulationsShareState, of->mastersComm); } if (mdof_flags & (MDOF_X | MDOF_V | MDOF_F)) diff --git a/src/gromacs/mdlib/tests/constrtestrunners.cpp b/src/gromacs/mdlib/tests/constrtestrunners.cpp index 1700e6e90b..7959cdd928 100644 --- a/src/gromacs/mdlib/tests/constrtestrunners.cpp +++ b/src/gromacs/mdlib/tests/constrtestrunners.cpp @@ -118,7 +118,7 @@ void applyLincs(ConstraintsTestData* testData, t_pbc pbc) cr.dd = nullptr; // Multi-sim record - gmx_multisim_t ms; + gmx_multisim_t ms{ 1, 0, MPI_COMM_NULL, MPI_COMM_NULL }; // Make blocka structure for faster LINCS setup std::vector> at2con_mt; diff --git a/src/gromacs/mdrun/replicaexchange.cpp b/src/gromacs/mdrun/replicaexchange.cpp index 9ff4b3817d..4c036dc2fb 100644 --- a/src/gromacs/mdrun/replicaexchange.cpp +++ b/src/gromacs/mdrun/replicaexchange.cpp @@ -3,7 +3,7 @@ * * Copyright (c) 1991-2000, University of Groningen, The Netherlands. * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2011-2019, by the GROMACS development team, led by + * Copyright (c) 2011-2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -166,12 +166,12 @@ static gmx_bool repl_quantity(const gmx_multisim_t* ms, struct gmx_repl_ex* re, gmx_bool bDiff; int s; - snew(qall, ms->nsim); + snew(qall, ms->numSimulations_); qall[re->repl] = q; - gmx_sum_sim(ms->nsim, qall, ms); + gmx_sum_sim(ms->numSimulations_, qall, ms); bDiff = FALSE; - for (s = 1; s < ms->nsim; s++) + for (s = 1; s < ms->numSimulations_; s++) { if (qall[s] != qall[0]) { @@ -185,7 +185,7 @@ static gmx_bool repl_quantity(const gmx_multisim_t* ms, struct gmx_repl_ex* re, re->type = ere; snew(re->q[ere], re->nrepl); - for (s = 0; s < ms->nsim; s++) + for (s = 0; s < ms->numSimulations_; s++) { re->q[ere][s] = qall[s]; } @@ -208,7 +208,7 @@ gmx_repl_ex_t init_replica_exchange(FILE* fplog, fprintf(fplog, "\nInitializing Replica Exchange\n"); - if (!isMultiSim(ms) || ms->nsim == 1) + if (!isMultiSim(ms) || ms->numSimulations_ == 1) { gmx_fatal(FARGS, "Nothing to exchange with only one replica, maybe you forgot to set the " @@ -235,8 +235,8 @@ gmx_repl_ex_t init_replica_exchange(FILE* fplog, snew(re, 1); - re->repl = ms->sim; - re->nrepl = ms->nsim; + re->repl = ms->simulationIndex_; + re->nrepl = ms->numSimulations_; snew(re->q, ereENDSINGLE); fprintf(fplog, "Repl There are %d replicas:\n", re->nrepl); @@ -507,14 +507,13 @@ static void exchange_reals(const gmx_multisim_t gmx_unused* ms, int gmx_unused b /* MPI_Sendrecv(v, n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0, buf,n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0, - ms->mpi_comm_masters,MPI_STATUS_IGNORE); + ms->mastersComm_,MPI_STATUS_IGNORE); */ { MPI_Request mpi_req; - MPI_Isend(v, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req); - MPI_Recv(buf, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, - MPI_STATUS_IGNORE); + MPI_Isend(v, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req); + MPI_Recv(buf, n * sizeof(real), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, MPI_STATUS_IGNORE); MPI_Wait(&mpi_req, MPI_STATUS_IGNORE); } #endif @@ -539,13 +538,13 @@ static void exchange_doubles(const gmx_multisim_t gmx_unused* ms, int gmx_unused /* MPI_Sendrecv(v, n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0, buf,n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0, - ms->mpi_comm_masters,MPI_STATUS_IGNORE); + ms->mastersComm_,MPI_STATUS_IGNORE); */ { MPI_Request mpi_req; - MPI_Isend(v, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req); - MPI_Recv(buf, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, + MPI_Isend(v, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req); + MPI_Recv(buf, n * sizeof(double), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, MPI_STATUS_IGNORE); MPI_Wait(&mpi_req, MPI_STATUS_IGNORE); } @@ -570,13 +569,13 @@ static void exchange_rvecs(const gmx_multisim_t gmx_unused* ms, int gmx_unused b /* MPI_Sendrecv(v[0], n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0, buf[0],n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0, - ms->mpi_comm_masters,MPI_STATUS_IGNORE); + ms->mastersComm_,MPI_STATUS_IGNORE); */ { MPI_Request mpi_req; - MPI_Isend(v[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, &mpi_req); - MPI_Recv(buf[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mpi_comm_masters, + MPI_Isend(v[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, &mpi_req); + MPI_Recv(buf[0], n * sizeof(rvec), MPI_BYTE, MSRANK(ms, b), 0, ms->mastersComm_, MPI_STATUS_IGNORE); MPI_Wait(&mpi_req, MPI_STATUS_IGNORE); } diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index 2e502f25dc..c9315bfc5c 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -832,8 +832,9 @@ int Mdrunner::mdrunner() physicalNodeComm = PhysicalNodeCommunicator(communicator, gmx_physicalnode_id_hash()); } - GMX_RELEASE_ASSERT(communicator == MPI_COMM_WORLD, "Must have valid world communicator"); - CommrecHandle crHandle = init_commrec(communicator, ms); + GMX_RELEASE_ASSERT(ms || communicator == MPI_COMM_WORLD, + "Must have valid world communicator unless running a multi-simulation"); + CommrecHandle crHandle = init_commrec(communicator); t_commrec* cr = crHandle.get(); GMX_RELEASE_ASSERT(cr != nullptr, "Must have valid commrec"); @@ -1248,7 +1249,7 @@ int Mdrunner::mdrunner() .appendTextFormatted( "This is simulation %d out of %d running as a composite GROMACS\n" "multi-simulation job. Setup for this simulation:\n", - ms->sim, ms->nsim); + ms->simulationIndex_, ms->numSimulations_); } GMX_LOG(mdlog.warning) .appendTextFormatted("Using %d MPI %s\n", cr->nnodes, diff --git a/src/gromacs/mdrun/simulationcontext.cpp b/src/gromacs/mdrun/simulationcontext.cpp index 91428c252b..0c421213bc 100644 --- a/src/gromacs/mdrun/simulationcontext.cpp +++ b/src/gromacs/mdrun/simulationcontext.cpp @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2018,2019, by the GROMACS development team, led by + * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -60,7 +60,9 @@ SimulationContext::SimulationContext(MPI_Comm communica "Without it, the communicator must be null."); if (!multiSimDirectoryNames.empty()) { - multiSimulation_ = std::make_unique(communicator, multiSimDirectoryNames); + multiSimulation_ = buildMultiSimulation(communicator, multiSimDirectoryNames); + // Use the communicator resulting from the split for the multi-simulation. + communicator_ = multiSimulation_->simulationComm_; } } diff --git a/src/gromacs/mdrunutility/handlerestart.cpp b/src/gromacs/mdrunutility/handlerestart.cpp index 220a92f365..b21f113029 100644 --- a/src/gromacs/mdrunutility/handlerestart.cpp +++ b/src/gromacs/mdrunutility/handlerestart.cpp @@ -533,7 +533,7 @@ checkpoint file was written). To help you identify which directories need attention, the %d simulations wanted the following respective behaviors: )", - ms->nsim); + ms->numSimulations_); for (index simIndex = 0; simIndex != ssize(startingBehaviors); ++simIndex) { auto behavior = static_cast(startingBehaviors[simIndex]); @@ -573,7 +573,7 @@ To help you identify which directories need attention, the %d simulation checkpoint files were from the following respective simulation parts: )", - ms->nsim); + ms->numSimulations_); for (index partIndex = 0; partIndex != ssize(simulationParts); ++partIndex) { message += formatString(" Simulation %6zd: %d\n", partIndex, simulationParts[partIndex]); diff --git a/src/gromacs/mdrunutility/multisim.cpp b/src/gromacs/mdrunutility/multisim.cpp index 290a1003a3..12b1f9dece 100644 --- a/src/gromacs/mdrunutility/multisim.cpp +++ b/src/gromacs/mdrunutility/multisim.cpp @@ -46,103 +46,116 @@ #include "config.h" #include "gromacs/mdtypes/commrec.h" +#include "gromacs/utility/exceptions.h" #include "gromacs/utility/fatalerror.h" #include "gromacs/utility/futil.h" #include "gromacs/utility/gmxassert.h" -#include "gromacs/utility/mpiinplacebuffers.h" #include "gromacs/utility/smalloc.h" -gmx_multisim_t::gmx_multisim_t() = default; - -gmx_multisim_t::gmx_multisim_t(MPI_Comm comm, gmx::ArrayRef multidirs) +std::unique_ptr buildMultiSimulation(MPI_Comm worldComm, + gmx::ArrayRef multidirs) { if (multidirs.empty()) { - return; + return nullptr; } if (!GMX_LIB_MPI && !multidirs.empty()) { - gmx_fatal(FARGS, - "mdrun -multidir is only supported when GROMACS has been " - "configured with a proper external MPI library."); + GMX_THROW(gmx::NotImplementedError( + "Multi-simulations are only supported when GROMACS has been " + "configured with a proper external MPI library.")); } if (multidirs.size() == 1) { /* NOTE: It would be nice if this special case worked, but this requires checks/tests. */ - gmx_fatal(FARGS, - "To run mdrun in multiple simulation mode, more then one " - "actual simulation is required. The single simulation case is not supported."); + GMX_THROW(gmx::NotImplementedError( + "To run mdrun in multi-simulation mode, more then one " + "actual simulation is required. The single simulation case is not supported.")); } -#if GMX_MPI +#if GMX_LIB_MPI int numRanks; - MPI_Comm_size(comm, &numRanks); + MPI_Comm_size(worldComm, &numRanks); if (numRanks % multidirs.size() != 0) { - gmx_fatal(FARGS, - "The number of ranks (%d) is not a multiple of the number of simulations (%td)", - numRanks, multidirs.ssize()); + auto message = gmx::formatString( + "The number of ranks (%d) is not a multiple of the number of simulations (%td)", + numRanks, multidirs.ssize()); + GMX_THROW(gmx::InconsistentInputError(message)); } - int numRanksPerSim = numRanks / multidirs.size(); - int rankWithinComm; - MPI_Comm_rank(comm, &rankWithinComm); + int numRanksPerSimulation = numRanks / multidirs.size(); + int rankWithinWorldComm; + MPI_Comm_rank(worldComm, &rankWithinWorldComm); if (debug) { fprintf(debug, "We have %td simulations, %d ranks per simulation, local simulation is %d\n", - multidirs.ssize(), numRanksPerSim, rankWithinComm / numRanksPerSim); + multidirs.ssize(), numRanksPerSimulation, rankWithinWorldComm / numRanksPerSimulation); } - nsim = multidirs.size(); - sim = rankWithinComm / numRanksPerSim; - /* Create a communicator for the master nodes */ - std::vector rank(nsim); - for (int i = 0; i < nsim; i++) + int numSimulations = multidirs.size(); + // Create a communicator for the master ranks of each simulation + std::vector ranksOfMasters(numSimulations); + for (int i = 0; i < numSimulations; i++) { - rank[i] = i * numRanksPerSim; + ranksOfMasters[i] = i * numRanksPerSimulation; + } + MPI_Group worldGroup; + // No need to free worldGroup later, we didn't create it. + MPI_Comm_group(worldComm, &worldGroup); + + MPI_Group mastersGroup = MPI_GROUP_NULL; + MPI_Group_incl(worldGroup, numSimulations, ranksOfMasters.data(), &mastersGroup); + MPI_Comm mastersComm = MPI_COMM_NULL; + MPI_Comm_create(worldComm, mastersGroup, &mastersComm); + if (mastersGroup != MPI_GROUP_NULL) + { + MPI_Group_free(&mastersGroup); } - MPI_Group mpi_group_world; - MPI_Comm_group(comm, &mpi_group_world); - MPI_Group_incl(mpi_group_world, nsim, rank.data(), &mpi_group_masters); - MPI_Comm_create(comm, mpi_group_masters, &mpi_comm_masters); - -# if !MPI_IN_PLACE_EXISTS - /* initialize the MPI_IN_PLACE replacement buffers */ - snew(mpb, 1); - mpb->ibuf = nullptr; - mpb->libuf = nullptr; - mpb->fbuf = nullptr; - mpb->dbuf = nullptr; - mpb->ibuf_alloc = 0; - mpb->libuf_alloc = 0; - mpb->fbuf_alloc = 0; - mpb->dbuf_alloc = 0; -# endif - // TODO This should throw upon error - gmx_chdir(multidirs[sim].c_str()); + int simulationIndex = rankWithinWorldComm / numRanksPerSimulation; + MPI_Comm simulationComm = MPI_COMM_NULL; + MPI_Comm_split(worldComm, simulationIndex, rankWithinWorldComm, &simulationComm); + + try + { + gmx_chdir(multidirs[simulationIndex].c_str()); + } + catch (gmx::GromacsException& e) + { + e.prependContext("While changing directory for multi-simulation to " + multidirs[simulationIndex]); + throw; + } + return std::make_unique(numSimulations, simulationIndex, mastersComm, simulationComm); #else - GMX_UNUSED_VALUE(comm); + GMX_UNUSED_VALUE(worldComm); + return nullptr; #endif } -gmx_multisim_t::~gmx_multisim_t() +gmx_multisim_t::gmx_multisim_t(int numSimulations, int simulationIndex, MPI_Comm mastersComm, MPI_Comm simulationComm) : + numSimulations_(numSimulations), + simulationIndex_(simulationIndex), + mastersComm_(mastersComm), + simulationComm_(simulationComm) { - done_mpi_in_place_buf(mpb); +} -#if GMX_MPI +gmx_multisim_t::~gmx_multisim_t() +{ +#if GMX_LIB_MPI // TODO This would work better if the result of MPI_Comm_split was // put into an RAII-style guard, such as gmx::unique_cptr. - if (mpi_comm_masters != MPI_COMM_NULL && mpi_comm_masters != MPI_COMM_WORLD) + if (mastersComm_ != MPI_COMM_NULL && mastersComm_ != MPI_COMM_WORLD) { - MPI_Comm_free(&mpi_comm_masters); + MPI_Comm_free(&mastersComm_); } - if (mpi_group_masters != MPI_GROUP_NULL) + if (simulationComm_ != MPI_COMM_NULL && simulationComm_ != MPI_COMM_WORLD) { - MPI_Group_free(&mpi_group_masters); + MPI_Comm_free(&simulationComm_); } #endif } @@ -198,7 +211,7 @@ void gmx_sumd_sim(int gmx_unused nr, double gmx_unused r[], const gmx_multisim_t #if !GMX_MPI GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumd_sim"); #else - gmx_sumd_comm(nr, r, ms->mpi_comm_masters); + gmx_sumd_comm(nr, r, ms->mastersComm_); #endif } @@ -207,7 +220,7 @@ void gmx_sumf_sim(int gmx_unused nr, float gmx_unused r[], const gmx_multisim_t #if !GMX_MPI GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumf_sim"); #else - gmx_sumf_comm(nr, r, ms->mpi_comm_masters); + gmx_sumf_comm(nr, r, ms->mastersComm_); #endif } @@ -217,21 +230,12 @@ void gmx_sumi_sim(int gmx_unused nr, int gmx_unused r[], const gmx_multisim_t gm GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumi_sim"); #else # if MPI_IN_PLACE_EXISTS - MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters); + MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mastersComm_); # else /* this is thread-unsafe, but it will do for now: */ - int i; - - if (nr > ms->mpb->ibuf_alloc) - { - ms->mpb->ibuf_alloc = nr; - srenew(ms->mpb->ibuf, ms->mpb->ibuf_alloc); - } - MPI_Allreduce(r, ms->mpb->ibuf, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters); - for (i = 0; i < nr; i++) - { - r[i] = ms->mpb->ibuf[i]; - } + ms->intBuffer.resize(nr); + MPI_Allreduce(r, ms->intBuffer.data(), ms->intBuffer.size(), MPI_INT, MPI_SUM, ms->mastersComm_); + std::copy(std::begin(ms->intBuffer), std::end(ms->intBuffer), r); # endif #endif } @@ -242,21 +246,12 @@ void gmx_sumli_sim(int gmx_unused nr, int64_t gmx_unused r[], const gmx_multisim GMX_RELEASE_ASSERT(false, "Invalid call to gmx_sumli_sim"); #else # if MPI_IN_PLACE_EXISTS - MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT64_T, MPI_SUM, ms->mpi_comm_masters); + MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT64_T, MPI_SUM, ms->mastersComm_); # else /* this is thread-unsafe, but it will do for now: */ - int i; - - if (nr > ms->mpb->libuf_alloc) - { - ms->mpb->libuf_alloc = nr; - srenew(ms->mpb->libuf, ms->mpb->libuf_alloc); - } - MPI_Allreduce(r, ms->mpb->libuf, nr, MPI_INT64_T, MPI_SUM, ms->mpi_comm_masters); - for (i = 0; i < nr; i++) - { - r[i] = ms->mpb->libuf[i]; - } + ms->int64Buffer.resize(nr); + MPI_Allreduce(r, ms->int64Buffer.data(), ms->int64Buffer.size(), MPI_INT64_T, MPI_SUM, ms->mastersComm_); + std::copy(std::begin(ms->int64Buffer), std::end(ms->int64Buffer), r); # endif #endif } @@ -267,9 +262,9 @@ std::vector gatherIntFromMultiSimulation(const gmx_multisim_t* ms, const in #if GMX_MPI if (ms != nullptr) { - valuesFromAllRanks.resize(ms->nsim); - valuesFromAllRanks[ms->sim] = localValue; - gmx_sumi_sim(ms->nsim, valuesFromAllRanks.data(), ms); + valuesFromAllRanks.resize(ms->numSimulations_); + valuesFromAllRanks[ms->simulationIndex_] = localValue; + gmx_sumi_sim(ms->numSimulations_, valuesFromAllRanks.data(), ms); } else { @@ -297,12 +292,12 @@ void check_multi_int(FILE* log, const gmx_multisim_t* ms, int val, const char* n gmx_fatal(FARGS, "check_multi_int called with a NULL communication pointer"); } - snew(ibuf, ms->nsim); - ibuf[ms->sim] = val; - gmx_sumi_sim(ms->nsim, ibuf, ms); + snew(ibuf, ms->numSimulations_); + ibuf[ms->simulationIndex_] = val; + gmx_sumi_sim(ms->numSimulations_, ibuf, ms); bCompatible = TRUE; - for (p = 1; p < ms->nsim; p++) + for (p = 1; p < ms->numSimulations_; p++) { bCompatible = bCompatible && (ibuf[p - 1] == ibuf[p]); } @@ -319,12 +314,12 @@ void check_multi_int(FILE* log, const gmx_multisim_t* ms, int val, const char* n if (nullptr != log) { fprintf(log, "\n%s is not equal for all subsystems\n", name); - for (p = 0; p < ms->nsim; p++) + for (p = 0; p < ms->numSimulations_; p++) { fprintf(log, " subsystem %d: %d\n", p, ibuf[p]); } } - gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim); + gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->numSimulations_); } sfree(ibuf); @@ -346,12 +341,12 @@ void check_multi_int64(FILE* log, const gmx_multisim_t* ms, int64_t val, const c gmx_fatal(FARGS, "check_multi_int called with a NULL communication pointer"); } - snew(ibuf, ms->nsim); - ibuf[ms->sim] = val; - gmx_sumli_sim(ms->nsim, ibuf, ms); + snew(ibuf, ms->numSimulations_); + ibuf[ms->simulationIndex_] = val; + gmx_sumli_sim(ms->numSimulations_, ibuf, ms); bCompatible = TRUE; - for (p = 1; p < ms->nsim; p++) + for (p = 1; p < ms->numSimulations_; p++) { bCompatible = bCompatible && (ibuf[p - 1] == ibuf[p]); } @@ -370,7 +365,7 @@ void check_multi_int64(FILE* log, const gmx_multisim_t* ms, int64_t val, const c if (nullptr != log) { fprintf(log, "\n%s is not equal for all subsystems\n", name); - for (p = 0; p < ms->nsim; p++) + for (p = 0; p < ms->numSimulations_; p++) { char strbuf[255]; /* first make the format string */ @@ -378,7 +373,7 @@ void check_multi_int64(FILE* log, const gmx_multisim_t* ms, int64_t val, const c fprintf(log, strbuf, p, ibuf[p]); } } - gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->nsim); + gmx_fatal(FARGS, "The %d subsystems are not compatible\n", ms->numSimulations_); } sfree(ibuf); @@ -391,9 +386,9 @@ bool findIsSimulationMasterRank(const gmx_multisim_t* ms, MPI_Comm communicator) // Ranks of multi-simulations know whether they are a master // rank. Ranks of non-multi simulation do not know until a // t_commrec is available. - if ((ms != nullptr) && (ms->nsim > 1)) + if ((ms != nullptr) && (ms->numSimulations_ > 1)) { - return ms->mpi_comm_masters != MPI_COMM_NULL; + return ms->mastersComm_ != MPI_COMM_NULL; } else { @@ -422,7 +417,7 @@ bool findIsSimulationMasterRank(const gmx_multisim_t* ms, MPI_Comm communicator) bool isMasterSim(const gmx_multisim_t* ms) { - return !isMultiSim(ms) || ms->sim == 0; + return !isMultiSim(ms) || ms->simulationIndex_ == 0; } bool isMasterSimMasterRank(const gmx_multisim_t* ms, const bool isMaster) diff --git a/src/gromacs/mdrunutility/multisim.h b/src/gromacs/mdrunutility/multisim.h index 2a31292a91..f616d527d1 100644 --- a/src/gromacs/mdrunutility/multisim.h +++ b/src/gromacs/mdrunutility/multisim.h @@ -45,10 +45,35 @@ #include #include +#include #include "gromacs/utility/arrayref.h" #include "gromacs/utility/gmxmpi.h" -#include "gromacs/utility/mpiinplacebuffers.h" + +struct gmx_multisim_t; + +/*! \libinternal + * \brief Builder function for gmx_multisim_t + * + * \param[in] worldComm MPI communicator to split when + * multi-simulation is requested. + * \param[in] multidirs Strings naming the subdirectories when + * multi-simulation is requested, otherwise empty + * + * Splits \c worldComm into \c multidirs.size() separate + * simulations, if >1, and creates a communication structure + * between the master ranks of these simulations. + * + * Valid to call regardless of build configuration, but \c + * multidirs must be empty unless a real MPI build is used. + * + * \throws NotImplementedError when \c multidirs is non-empty unless using real MPI is true + * \throws NotImplementedError when \c multidirs has exactly one element + * \throws InconsistentInputError when the number of MPI ranks is not a multiple of the number of \c multidirs + * \throws FileIOError when the simulation cannot change to the working directory in \c multidirs + */ +std::unique_ptr buildMultiSimulation(MPI_Comm worldComm, + gmx::ArrayRef multidirs); /*! \libinternal * \brief Coordinate multi-simulation resources for mdrun @@ -57,30 +82,45 @@ */ struct gmx_multisim_t { - //! Default constructor - gmx_multisim_t(); - /*! \brief Constructor useful for mdrun simulations + /*! \brief Constructor * - * Splits the communicator into multidirs.size() separate - * simulations, if >1, and creates a communication structure - * between the master these simulations. + * \param[in] numSimulations The number of simulations in the MPI world. + * \param[in] simulationIndex The index of this simulation in the set of simulations. + * \param[in] mastersComm On master ranks, the communicator among master ranks; + * otherwise MPI_COMM_NULL. + * \param[in] simulationComm The communicator among ranks of this simulation. * - * Valid to call regardless of build configuration, but \c - * multidirs must be empty unless a real MPI build is used. */ - gmx_multisim_t(MPI_Comm comm, gmx::ArrayRef multidirs); + * Assumes ownership of the communicators if they are neither + * MPI_COMM_WORLD nor MPI_COMM_NULL. If so, upon destruction will + * call MPI_Comm_free on them. + */ + gmx_multisim_t(int numSimulations, int simulationIndex, MPI_Comm mastersComm, MPI_Comm simulationComm); //! Destructor ~gmx_multisim_t(); //! The number of simulations in the set of multi-simulations - int nsim = 1; + int numSimulations_ = 1; //! The index of the simulation that owns this object within the set - int sim = 0; - //! The MPI Group between master ranks of simulations, valid only on master ranks. - MPI_Group mpi_group_masters = MPI_GROUP_NULL; + int simulationIndex_ = 0; //! The MPI communicator between master ranks of simulations, valid only on master ranks. - MPI_Comm mpi_comm_masters = MPI_COMM_NULL; - //! Communication buffers needed if MPI_IN_PLACE isn't supported - mpi_in_place_buf_t* mpb = nullptr; + MPI_Comm mastersComm_ = MPI_COMM_NULL; + //! The MPI communicator between ranks of this simulation. + MPI_Comm simulationComm_ = MPI_COMM_NULL; + /*! \brief Communication buffers needed if MPI_IN_PLACE isn't supported + * + * Other types could be added as needed. + * + * These vectors are unused when MPI_IN_PLACE is available + * and could be removed with preprocessing (or perhaps + * templating) or simply requiring MPI 2.0 (the standard + * introduced in 1997). However, the additional cache pressure + * introduced by the extra size of this type is not of great + * concern, since we have at most one per MPI rank. + * See issue #3591. */ + //! \{ + std::vector intBuffer_; + std::vector int64Buffer_; + //! \} }; //! Calculate the sum over the simulations of an array of ints diff --git a/src/gromacs/taskassignment/resourcedivision.cpp b/src/gromacs/taskassignment/resourcedivision.cpp index 04bdb2f0b1..6d061c94e5 100644 --- a/src/gromacs/taskassignment/resourcedivision.cpp +++ b/src/gromacs/taskassignment/resourcedivision.cpp @@ -909,7 +909,7 @@ void checkAndUpdateRequestedNumOpenmpThreads(gmx_hw_opt_t* hw_opt, * all detected ncore_tot physical cores. We are currently not * checking for that here. */ - int numRanksTot = cr->nnodes * (isMultiSim(ms) ? ms->nsim : 1); + int numRanksTot = cr->nnodes * (isMultiSim(ms) ? ms->numSimulations_ : 1); int numAtomsPerRank = mtop.natoms / cr->nnodes; int numCoresPerRank = hwinfo.ncore_tot / numRanksTot; if (numAtomsPerRank < c_numAtomsPerCoreSquaredSmtThreshold * gmx::square(numCoresPerRank)) diff --git a/src/gromacs/tools/pme_error.cpp b/src/gromacs/tools/pme_error.cpp index cb8087f6e7..ba1fb3b142 100644 --- a/src/gromacs/tools/pme_error.cpp +++ b/src/gromacs/tools/pme_error.cpp @@ -1121,7 +1121,7 @@ int gmx_pme_error(int argc, char* argv[]) #define NFILE asize(fnm) - CommrecHandle commrecHandle = init_commrec(MPI_COMM_WORLD, nullptr); + CommrecHandle commrecHandle = init_commrec(MPI_COMM_WORLD); t_commrec* cr = commrecHandle.get(); PCA_Flags = PCA_NOEXIT_ON_ARGS; diff --git a/src/gromacs/utility/futil.cpp b/src/gromacs/utility/futil.cpp index 15c986a8d8..b09c5403b8 100644 --- a/src/gromacs/utility/futil.cpp +++ b/src/gromacs/utility/futil.cpp @@ -743,7 +743,9 @@ void gmx_chdir(const char* directory) #endif if (rc != 0) { - gmx_fatal(FARGS, "Cannot change directory to '%s'. Reason: %s", directory, strerror(errno)); + auto message = gmx::formatString("Cannot change directory to '%s'. Reason: %s", directory, + strerror(errno)); + GMX_THROW(gmx::FileIOError(message)); } } diff --git a/src/gromacs/utility/mpiinplacebuffers.cpp b/src/gromacs/utility/mpiinplacebuffers.cpp deleted file mode 100644 index a35c4181fd..0000000000 --- a/src/gromacs/utility/mpiinplacebuffers.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * This file is part of the GROMACS molecular simulation package. - * - * Copyright (c) 2019, by the GROMACS development team, led by - * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, - * and including many others, as listed in the AUTHORS file in the - * top-level source directory and at http://www.gromacs.org. - * - * GROMACS is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * GROMACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with GROMACS; if not, see - * http://www.gnu.org/licenses, or write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * If you want to redistribute modifications to GROMACS, please - * consider that scientific software is very special. Version - * control is crucial - bugs must be traceable. We will be happy to - * consider code for inclusion in the official distribution, but - * derived work must not be called official GROMACS. Details are found - * in the README & COPYING files - if they are missing, get the - * official version at http://www.gromacs.org. - * - * To help us fund GROMACS development, we humbly ask that you cite - * the research papers on the package. Check out http://www.gromacs.org. - */ -/*! \internal \file - * \brief Defines cleanup function for the copies necessary when - * MPI_IN_PLACE is not supported by the MPI library. - * - * \author Mark Abraham - * \ingroup module_utility - */ -#include "gmxpre.h" - -#include "mpiinplacebuffers.h" - -#include "gromacs/utility/smalloc.h" - -void done_mpi_in_place_buf(mpi_in_place_buf_t* buf) -{ - if (nullptr != buf) - { - sfree(buf->ibuf); - sfree(buf->libuf); - sfree(buf->fbuf); - sfree(buf->dbuf); - sfree(buf); - } -} diff --git a/src/gromacs/utility/mpiinplacebuffers.h b/src/gromacs/utility/mpiinplacebuffers.h deleted file mode 100644 index dfbac0de09..0000000000 --- a/src/gromacs/utility/mpiinplacebuffers.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * This file is part of the GROMACS molecular simulation package. - * - * Copyright (c) 2019, by the GROMACS development team, led by - * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, - * and including many others, as listed in the AUTHORS file in the - * top-level source directory and at http://www.gromacs.org. - * - * GROMACS is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * GROMACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with GROMACS; if not, see - * http://www.gnu.org/licenses, or write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * If you want to redistribute modifications to GROMACS, please - * consider that scientific software is very special. Version - * control is crucial - bugs must be traceable. We will be happy to - * consider code for inclusion in the official distribution, but - * derived work must not be called official GROMACS. Details are found - * in the README & COPYING files - if they are missing, get the - * official version at http://www.gromacs.org. - * - * To help us fund GROMACS development, we humbly ask that you cite - * the research papers on the package. Check out http://www.gromacs.org. - */ -/*! \libinternal \file - * \brief Declares object that provides buffers for the copies - * necessary when MPI_IN_PLACE is not supported by the MPI library. - * - * The struct is declared in the header so that functionality it - * supports can be inlined. - * - * \author Mark Abraham - * \inlibraryapi - * \ingroup module_utility - */ -#ifndef GMX_UTILTIY_MPIINPLACEBUFFERS_H -#define GMX_UTILTIY_MPIINPLACEBUFFERS_H - -#include - -struct mpi_in_place_buf_t -{ - /* these buffers are used as destination buffers if MPI_IN_PLACE isn't - supported.*/ - int* ibuf; /* for ints */ - int ibuf_alloc; - - int64_t* libuf; - int libuf_alloc; - - float* fbuf; /* for floats */ - int fbuf_alloc; - - double* dbuf; /* for doubles */ - int dbuf_alloc; -}; - -//! Cleans up the buffers -void done_mpi_in_place_buf(mpi_in_place_buf_t* buf); - -#endif -- 2.22.0