2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team.
6 * Copyright (c) 2013,2014,2015,2016,2017 by the GROMACS development team.
7 * Copyright (c) 2018,2019,2020,2021, by the GROMACS development team, led by
8 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
9 * and including many others, as listed in the AUTHORS file in the
10 * top-level source directory and at http://www.gromacs.org.
12 * GROMACS is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public License
14 * as published by the Free Software Foundation; either version 2.1
15 * of the License, or (at your option) any later version.
17 * GROMACS is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Lesser General Public License for more details.
22 * You should have received a copy of the GNU Lesser General Public
23 * License along with GROMACS; if not, see
24 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
25 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
27 * If you want to redistribute modifications to GROMACS, please
28 * consider that scientific software is very special. Version
29 * control is crucial - bugs must be traceable. We will be happy to
30 * consider code for inclusion in the official distribution, but
31 * derived work must not be called official GROMACS. Details are found
32 * in the README & COPYING files - if they are missing, get the
33 * official version at http://www.gromacs.org.
35 * To help us fund GROMACS development, we humbly ask that you cite
36 * the research papers on the package. Check out http://www.gromacs.org.
40 * \brief This file contains function definitions necessary for
41 * managing the offload of long-ranged PME work to separate MPI rank,
42 * for computing energies and forces (Coulomb and LJ).
44 * \author Berk Hess <hess@kth.se>
45 * \ingroup module_ewald
57 #include "gromacs/domdec/domdec.h"
58 #include "gromacs/domdec/domdec_struct.h"
59 #include "gromacs/ewald/pme.h"
60 #include "gromacs/ewald/pme_pp_comm_gpu.h"
61 #include "gromacs/gmxlib/network.h"
62 #include "gromacs/math/vec.h"
63 #include "gromacs/mdlib/gmx_omp_nthreads.h"
64 #include "gromacs/mdtypes/commrec.h"
65 #include "gromacs/mdtypes/forceoutput.h"
66 #include "gromacs/mdtypes/forcerec.h"
67 #include "gromacs/mdtypes/interaction_const.h"
68 #include "gromacs/mdtypes/md_enums.h"
69 #include "gromacs/mdtypes/state_propagator_data_gpu.h"
70 #include "gromacs/nbnxm/nbnxm.h"
71 #include "gromacs/timing/wallcycle.h"
72 #include "gromacs/utility/fatalerror.h"
73 #include "gromacs/utility/gmxmpi.h"
74 #include "gromacs/utility/smalloc.h"
76 #include "pme_pp_communication.h"
78 /*! \brief Block to wait for communication to PME ranks to complete
80 * This should be faster with a real non-blocking MPI implementation
82 static constexpr bool c_useDelayedWait = false;
84 /*! \brief Wait for the pending data send requests to PME ranks to complete */
85 static void gmx_pme_send_coeffs_coords_wait(gmx_domdec_t* dd)
90 MPI_Waitall(dd->nreq_pme, dd->req_pme, MPI_STATUSES_IGNORE);
96 /*! \brief Send data to PME ranks */
97 static void gmx_pme_send_coeffs_coords(t_forcerec* fr,
100 gmx::ArrayRef<const real> chargeA,
101 gmx::ArrayRef<const real> chargeB,
102 gmx::ArrayRef<const real> c6A,
103 gmx::ArrayRef<const real> c6B,
104 gmx::ArrayRef<const real> sigmaA,
105 gmx::ArrayRef<const real> sigmaB,
107 gmx::ArrayRef<const gmx::RVec> x,
113 bool useGpuPmePpComms,
114 bool reinitGpuPmePpComms,
115 bool sendCoordinatesFromGpu,
116 bool receiveForcesToGpu,
117 GpuEventSynchronizer* coordinatesReadyOnDeviceEvent)
120 gmx_pme_comm_n_box_t* cnb;
124 n = dd_numHomeAtoms(*dd);
129 "PP rank %d sending to PME rank %d: %d%s%s%s%s\n",
133 (flags & PP_PME_CHARGE) ? " charges" : "",
134 (flags & PP_PME_SQRTC6) ? " sqrtC6" : "",
135 (flags & PP_PME_SIGMA) ? " sigma" : "",
136 (flags & PP_PME_COORD) ? " coordinates" : "");
139 if (useGpuPmePpComms)
141 flags |= PP_PME_GPUCOMMS;
142 if (receiveForcesToGpu)
144 flags |= PP_PME_RECVFTOGPU;
148 if (c_useDelayedWait)
150 /* We can not use cnb until pending communication has finished */
151 gmx_pme_send_coeffs_coords_wait(dd);
154 if (dd->pme_receive_vir_ener)
156 /* Peer PP node: communicate all data */
157 if (dd->cnb == nullptr)
165 cnb->maxshift_x = maxshift_x;
166 cnb->maxshift_y = maxshift_y;
167 cnb->lambda_q = lambda_q;
168 cnb->lambda_lj = lambda_lj;
170 if (flags & PP_PME_COORD)
172 copy_mat(box, cnb->box);
181 &dd->req_pme[dd->nreq_pme++]);
184 else if (flags & (PP_PME_CHARGE | PP_PME_SQRTC6 | PP_PME_SIGMA))
187 /* Communicate only the number of atoms */
194 &dd->req_pme[dd->nreq_pme++]);
201 if (flags & PP_PME_CHARGE)
203 MPI_Isend(chargeA.data(),
209 &dd->req_pme[dd->nreq_pme++]);
211 if (flags & PP_PME_CHARGEB)
213 MPI_Isend(chargeB.data(),
219 &dd->req_pme[dd->nreq_pme++]);
221 if (flags & PP_PME_SQRTC6)
223 MPI_Isend(c6A.data(),
229 &dd->req_pme[dd->nreq_pme++]);
231 if (flags & PP_PME_SQRTC6B)
233 MPI_Isend(c6B.data(),
239 &dd->req_pme[dd->nreq_pme++]);
241 if (flags & PP_PME_SIGMA)
243 MPI_Isend(sigmaA.data(),
249 &dd->req_pme[dd->nreq_pme++]);
251 if (flags & PP_PME_SIGMAB)
253 MPI_Isend(sigmaB.data(),
259 &dd->req_pme[dd->nreq_pme++]);
261 if (flags & PP_PME_COORD)
263 if (reinitGpuPmePpComms)
265 std::vector<gmx::RVec>& buffer = cr->dd->pmeForceReceiveBuffer;
267 fr->pmePpCommGpu->reinit(n);
270 if (useGpuPmePpComms && (fr != nullptr))
272 if (sendCoordinatesFromGpu)
274 fr->pmePpCommGpu->sendCoordinatesToPmeFromGpu(
275 fr->stateGpu->getCoordinates(), n, coordinatesReadyOnDeviceEvent);
279 fr->pmePpCommGpu->sendCoordinatesToPmeFromCpu(
280 const_cast<gmx::RVec*>(x.data()), n, coordinatesReadyOnDeviceEvent);
291 &dd->req_pme[dd->nreq_pme++]);
296 GMX_UNUSED_VALUE(fr);
297 GMX_UNUSED_VALUE(chargeA);
298 GMX_UNUSED_VALUE(chargeB);
299 GMX_UNUSED_VALUE(c6A);
300 GMX_UNUSED_VALUE(c6B);
301 GMX_UNUSED_VALUE(sigmaA);
302 GMX_UNUSED_VALUE(sigmaB);
304 GMX_UNUSED_VALUE(reinitGpuPmePpComms);
305 GMX_UNUSED_VALUE(sendCoordinatesFromGpu);
306 GMX_UNUSED_VALUE(coordinatesReadyOnDeviceEvent);
308 if (!c_useDelayedWait)
310 /* Wait for the data to arrive */
311 /* We can skip this wait as we are sure x and q will not be modified
312 * before the next call to gmx_pme_send_x_q or gmx_pme_receive_f.
314 gmx_pme_send_coeffs_coords_wait(dd);
318 void gmx_pme_send_parameters(const t_commrec* cr,
319 const interaction_const_t& interactionConst,
322 gmx::ArrayRef<const real> chargeA,
323 gmx::ArrayRef<const real> chargeB,
324 gmx::ArrayRef<const real> sqrt_c6A,
325 gmx::ArrayRef<const real> sqrt_c6B,
326 gmx::ArrayRef<const real> sigmaA,
327 gmx::ArrayRef<const real> sigmaB,
331 unsigned int flags = 0;
333 if (EEL_PME(interactionConst.eeltype))
335 flags |= PP_PME_CHARGE;
337 if (EVDW_PME(interactionConst.vdwtype))
339 flags |= (PP_PME_SQRTC6 | PP_PME_SIGMA);
341 if (bFreeEnergy_q || bFreeEnergy_lj)
343 /* Assumes that the B state flags are in the bits just above
344 * the ones for the A state. */
345 flags |= (flags << 1);
348 gmx_pme_send_coeffs_coords(nullptr,
358 gmx::ArrayRef<gmx::RVec>(),
371 void gmx_pme_send_coordinates(t_forcerec* fr,
374 gmx::ArrayRef<const gmx::RVec> x,
377 bool computeEnergyAndVirial,
379 bool useGpuPmePpComms,
380 bool receiveCoordinateAddressFromPme,
381 bool sendCoordinatesFromGpu,
382 bool receiveForcesToGpu,
383 GpuEventSynchronizer* coordinatesReadyOnDeviceEvent,
384 gmx_wallcycle* wcycle)
386 wallcycle_start(wcycle, WallCycleCounter::PpPmeSendX);
388 unsigned int flags = PP_PME_COORD;
389 if (computeEnergyAndVirial)
391 flags |= PP_PME_ENER_VIR;
393 gmx_pme_send_coeffs_coords(fr,
410 receiveCoordinateAddressFromPme,
411 sendCoordinatesFromGpu,
413 coordinatesReadyOnDeviceEvent);
415 wallcycle_stop(wcycle, WallCycleCounter::PpPmeSendX);
418 void gmx_pme_send_finish(const t_commrec* cr)
420 unsigned int flags = PP_PME_FINISH;
422 gmx_pme_send_coeffs_coords(
423 nullptr, cr, flags, {}, {}, {}, {}, {}, {}, nullptr, gmx::ArrayRef<gmx::RVec>(), 0, 0, 0, 0, -1, false, false, false, false, nullptr);
426 void gmx_pme_send_switchgrid(const t_commrec* cr, ivec grid_size, real ewaldcoeff_q, real ewaldcoeff_lj)
429 gmx_pme_comm_n_box_t cnb;
431 /* Only let one PP node signal each PME node */
432 if (cr->dd->pme_receive_vir_ener)
434 cnb.flags = PP_PME_SWITCHGRID;
435 copy_ivec(grid_size, cnb.grid_size);
436 cnb.ewaldcoeff_q = ewaldcoeff_q;
437 cnb.ewaldcoeff_lj = ewaldcoeff_lj;
439 /* We send this, uncommon, message blocking to simplify the code */
440 MPI_Send(&cnb, sizeof(cnb), MPI_BYTE, cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
443 GMX_UNUSED_VALUE(cr);
444 GMX_UNUSED_VALUE(grid_size);
445 GMX_UNUSED_VALUE(ewaldcoeff_q);
446 GMX_UNUSED_VALUE(ewaldcoeff_lj);
450 void gmx_pme_send_resetcounters(const t_commrec gmx_unused* cr, int64_t gmx_unused step)
453 gmx_pme_comm_n_box_t cnb;
455 /* Only let one PP node signal each PME node */
456 if (cr->dd->pme_receive_vir_ener)
458 cnb.flags = PP_PME_RESETCOUNTERS;
461 /* We send this, uncommon, message blocking to simplify the code */
462 MPI_Send(&cnb, sizeof(cnb), MPI_BYTE, cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
467 /*! \brief Receive virial and energy from PME rank */
468 static void receive_virial_energy(const t_commrec* cr,
469 gmx::ForceWithVirial* forceWithVirial,
476 gmx_pme_comm_vir_ene_t cve;
478 if (cr->dd->pme_receive_vir_ener)
483 "PP rank %d receiving from PME rank %d: virial and energy\n",
488 MPI_Recv(&cve, sizeof(cve), MPI_BYTE, cr->dd->pme_nodeid, 1, cr->mpi_comm_mysim, MPI_STATUS_IGNORE);
490 memset(&cve, 0, sizeof(cve));
493 forceWithVirial->addVirialContribution(cve.vir_q);
494 forceWithVirial->addVirialContribution(cve.vir_lj);
495 *energy_q = cve.energy_q;
496 *energy_lj = cve.energy_lj;
497 *dvdlambda_q += cve.dvdlambda_q;
498 *dvdlambda_lj += cve.dvdlambda_lj;
499 *pme_cycles = cve.cycles;
501 if (cve.stop_cond != StopCondition::None)
503 gmx_set_stop_condition(cve.stop_cond);
514 /*! \brief Recieve force data from PME ranks */
515 static void recvFFromPme(gmx::PmePpCommGpu* pmePpCommGpu,
519 bool useGpuPmePpComms,
520 bool receivePmeForceToGpu)
522 if (useGpuPmePpComms)
524 GMX_ASSERT(pmePpCommGpu != nullptr, "Need valid pmePpCommGpu");
525 // Receive forces from PME rank
526 pmePpCommGpu->receiveForceFromPme(static_cast<gmx::RVec*>(recvptr), n, receivePmeForceToGpu);
530 // Receive data using MPI
532 MPI_Recv(recvptr, n * sizeof(rvec), MPI_BYTE, cr->dd->pme_nodeid, 0, cr->mpi_comm_mysim, MPI_STATUS_IGNORE);
534 GMX_UNUSED_VALUE(cr);
541 void gmx_pme_receive_f(gmx::PmePpCommGpu* pmePpCommGpu,
543 gmx::ForceWithVirial* forceWithVirial,
548 bool useGpuPmePpComms,
549 bool receivePmeForceToGpu,
552 if (c_useDelayedWait)
554 /* Wait for the x request to finish */
555 gmx_pme_send_coeffs_coords_wait(cr->dd);
558 const int natoms = dd_numHomeAtoms(*cr->dd);
559 std::vector<gmx::RVec>& buffer = cr->dd->pmeForceReceiveBuffer;
560 buffer.resize(natoms);
562 void* recvptr = reinterpret_cast<void*>(buffer.data());
563 recvFFromPme(pmePpCommGpu, recvptr, natoms, cr, useGpuPmePpComms, receivePmeForceToGpu);
565 int nt = gmx_omp_nthreads_get_simple_rvec_task(ModuleMultiThread::Default, natoms);
567 gmx::ArrayRef<gmx::RVec> f = forceWithVirial->force_;
569 if (!receivePmeForceToGpu)
571 /* Note that we would like to avoid this conditional by putting it
572 * into the omp pragma instead, but then we still take the full
573 * omp parallel for overhead (at least with gcc5).
577 for (int i = 0; i < natoms; i++)
584 #pragma omp parallel for num_threads(nt) schedule(static)
585 for (int i = 0; i < natoms; i++)
592 receive_virial_energy(cr, forceWithVirial, energy_q, energy_lj, dvdlambda_q, dvdlambda_lj, pme_cycles);