From 7c6cf6db799d93220b76057657b76316ad86bbe5 Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Thu, 31 Oct 2019 20:20:01 +0100 Subject: [PATCH] Add haveEwaldSurfaceContribution to SimulationWorkload When update is offloaded, the coordinates are needed on host if an Ewald surface contribution is to be computed. This flag is set to reflect that and used when the decision of whether the coordinates are to be copied on this step is made. Change-Id: Iba40387f065e6fae2551540136e925f7c764d6aa --- src/gromacs/mdlib/force.cpp | 10 ++--- src/gromacs/mdlib/sim_util.cpp | 5 +-- src/gromacs/mdrun/runner.cpp | 3 +- src/gromacs/mdtypes/inputrec.cpp | 6 +++ src/gromacs/mdtypes/inputrec.h | 4 ++ src/gromacs/mdtypes/simulation_workload.h | 2 + .../decidesimulationworkload.cpp | 38 ++++++++++--------- .../taskassignment/decidesimulationworkload.h | 30 ++++++++------- 8 files changed, 56 insertions(+), 42 deletions(-) diff --git a/src/gromacs/mdlib/force.cpp b/src/gromacs/mdlib/force.cpp index 99f0b97a21..a82e5a98af 100644 --- a/src/gromacs/mdlib/force.cpp +++ b/src/gromacs/mdlib/force.cpp @@ -195,16 +195,14 @@ do_force_lowlevel(t_forcerec *fr, thisRankHasDuty(cr, DUTY_PME) && (pme_run_mode(fr->pmedata) == PmeRunMode::CPU); - const bool haveEwaldSurfaceTerms = - EEL_PME_EWALD(fr->ic->eeltype) && - (ir->ewald_geometry != eewg3D || ir->epsilon_surface != 0); + const bool haveEwaldSurfaceTerm = haveEwaldSurfaceContribution(*ir); /* Do long-range electrostatics and/or LJ-PME * and compute PME surface terms when necessary. */ if (computePmeOnCpu || fr->ic->eeltype == eelEWALD || - haveEwaldSurfaceTerms) + haveEwaldSurfaceTerm) { int status = 0; real Vlr_q = 0, Vlr_lj = 0; @@ -217,8 +215,8 @@ do_force_lowlevel(t_forcerec *fr, if (EEL_PME_EWALD(fr->ic->eeltype) || EVDW_PME(fr->ic->vdwtype)) { - /* Calculate Ewald surface terms, when necessary */ - if (haveEwaldSurfaceTerms) + /* Calculate the Ewald surface force and energy contributions, when necessary */ + if (haveEwaldSurfaceTerm) { wallcycle_sub_start(wcycle, ewcsEWALD_CORRECTION); diff --git a/src/gromacs/mdlib/sim_util.cpp b/src/gromacs/mdlib/sim_util.cpp index 6a2cfbe34e..f5a73578ba 100644 --- a/src/gromacs/mdlib/sim_util.cpp +++ b/src/gromacs/mdlib/sim_util.cpp @@ -799,10 +799,9 @@ setupDomainLifetimeWorkload(const t_inputrec &inputrec, // We assume we have local force work if there are CPU // force tasks including PME or nonbondeds. domainWork.haveCpuLocalForceWork = domainWork.haveSpecialForces || domainWork.haveCpuListedForceWork || domainWork.haveFreeEnergyWork || - simulationWork.useCpuNonbonded || simulationWork.useCpuPme || - (EEL_PME_EWALD(inputrec.coulombtype) && (inputrec.ewald_geometry == eewg3DC || - inputrec.epsilon_surface != 0)) || + simulationWork.useCpuNonbonded || simulationWork.useCpuPme || simulationWork.haveEwaldSurfaceContribution || inputrec.nwall > 0; + return domainWork; } diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index 1c2fb3e42d..dc647f6361 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -1583,7 +1583,8 @@ int Mdrunner::mdrunner() useGpuForUpdate, devFlags.enableGpuBufferOps, devFlags.enableGpuHaloExchange, - devFlags.enableGpuPmePPComm); + devFlags.enableGpuPmePPComm, + haveEwaldSurfaceContribution(*inputrec)); std::unique_ptr stateGpu; if (gpusWereDetected && ((useGpuForPme && thisRankHasDuty(cr, DUTY_PME)) || runScheduleWork.simulationWork.useGpuBufferOps)) diff --git a/src/gromacs/mdtypes/inputrec.cpp b/src/gromacs/mdtypes/inputrec.cpp index 7445dfc4cc..354d39f8ea 100644 --- a/src/gromacs/mdtypes/inputrec.cpp +++ b/src/gromacs/mdtypes/inputrec.cpp @@ -1549,3 +1549,9 @@ real maxReferenceTemperature(const t_inputrec &ir) return maxTemperature; } + +bool haveEwaldSurfaceContribution(const t_inputrec &ir) +{ + return EEL_PME_EWALD(ir.coulombtype) && (ir.ewald_geometry == eewg3DC || + ir.epsilon_surface != 0); +} diff --git a/src/gromacs/mdtypes/inputrec.h b/src/gromacs/mdtypes/inputrec.h index 6cff9d6df6..86dc181cd1 100644 --- a/src/gromacs/mdtypes/inputrec.h +++ b/src/gromacs/mdtypes/inputrec.h @@ -693,4 +693,8 @@ int ndof_com(const t_inputrec *ir); */ real maxReferenceTemperature(const t_inputrec &ir); +/*! \brief Returns whether there is an Ewald surface contribution + */ +bool haveEwaldSurfaceContribution(const t_inputrec &ir); + #endif /* GMX_MDTYPES_INPUTREC_H */ diff --git a/src/gromacs/mdtypes/simulation_workload.h b/src/gromacs/mdtypes/simulation_workload.h index 6594081c50..6100e70e6d 100644 --- a/src/gromacs/mdtypes/simulation_workload.h +++ b/src/gromacs/mdtypes/simulation_workload.h @@ -158,6 +158,8 @@ class SimulationWorkload bool useGpuPmePpCommunication = false; //! If direct GPU-GPU communication is enabled. bool useGpuDirectCommunication = false; + //! If there is an Ewald surface (dipole) term to compute + bool haveEwaldSurfaceContribution = false; }; class MdrunScheduleWorkload diff --git a/src/gromacs/taskassignment/decidesimulationworkload.cpp b/src/gromacs/taskassignment/decidesimulationworkload.cpp index ea82f95f42..933aeaef67 100644 --- a/src/gromacs/taskassignment/decidesimulationworkload.cpp +++ b/src/gromacs/taskassignment/decidesimulationworkload.cpp @@ -50,26 +50,28 @@ namespace gmx { -SimulationWorkload createSimulationWorkload(bool useGpuForNonbonded, - PmeRunMode pmeRunMode, - bool useGpuForBonded, - bool useGpuForUpdate, - bool useGpuForBufferOps, - bool useGpuHaloExchange, - bool useGpuPmePpComm) +SimulationWorkload createSimulationWorkload(bool useGpuForNonbonded, + PmeRunMode pmeRunMode, + bool useGpuForBonded, + bool useGpuForUpdate, + bool useGpuForBufferOps, + bool useGpuHaloExchange, + bool useGpuPmePpComm, + bool haveEwaldSurfaceContribution) { SimulationWorkload simulationWorkload; - simulationWorkload.useCpuNonbonded = !useGpuForNonbonded; - simulationWorkload.useGpuNonbonded = useGpuForNonbonded; - simulationWorkload.useCpuPme = (pmeRunMode == PmeRunMode::CPU); - simulationWorkload.useGpuPme = (pmeRunMode == PmeRunMode::GPU || pmeRunMode == PmeRunMode::Mixed); - simulationWorkload.useGpuPmeFft = (pmeRunMode == PmeRunMode::Mixed); - simulationWorkload.useGpuBonded = useGpuForBonded; - simulationWorkload.useGpuUpdate = useGpuForUpdate; - simulationWorkload.useGpuBufferOps = useGpuForBufferOps || useGpuForUpdate; - simulationWorkload.useGpuHaloExchange = useGpuHaloExchange; - simulationWorkload.useGpuPmePpCommunication = useGpuPmePpComm; - simulationWorkload.useGpuDirectCommunication = useGpuHaloExchange || useGpuPmePpComm; + simulationWorkload.useCpuNonbonded = !useGpuForNonbonded; + simulationWorkload.useGpuNonbonded = useGpuForNonbonded; + simulationWorkload.useCpuPme = (pmeRunMode == PmeRunMode::CPU); + simulationWorkload.useGpuPme = (pmeRunMode == PmeRunMode::GPU || pmeRunMode == PmeRunMode::Mixed); + simulationWorkload.useGpuPmeFft = (pmeRunMode == PmeRunMode::Mixed); + simulationWorkload.useGpuBonded = useGpuForBonded; + simulationWorkload.useGpuUpdate = useGpuForUpdate; + simulationWorkload.useGpuBufferOps = useGpuForBufferOps || useGpuForUpdate; + simulationWorkload.useGpuHaloExchange = useGpuHaloExchange; + simulationWorkload.useGpuPmePpCommunication = useGpuPmePpComm; + simulationWorkload.useGpuDirectCommunication = useGpuHaloExchange || useGpuPmePpComm; + simulationWorkload.haveEwaldSurfaceContribution = haveEwaldSurfaceContribution; return simulationWorkload; } diff --git a/src/gromacs/taskassignment/decidesimulationworkload.h b/src/gromacs/taskassignment/decidesimulationworkload.h index b389da53c1..ef53314b39 100644 --- a/src/gromacs/taskassignment/decidesimulationworkload.h +++ b/src/gromacs/taskassignment/decidesimulationworkload.h @@ -44,6 +44,7 @@ #include +#include "gromacs/mdtypes/inputrec.h" #include "gromacs/mdtypes/simulation_workload.h" enum class PmeRunMode; @@ -55,25 +56,26 @@ namespace gmx * Build datastructure that contains decisions whether to run different workload * task on GPUs. * - * \param[in] useGpuForNonbonded If we have short-range nonbonded interactions + * \param[in] useGpuForNonbonded Whether we have short-range nonbonded interactions * calculations on GPU(s). * \param[in] pmeRunMode Run mode indicating what resource is PME execured on. - * \param[in] useGpuForBonded If bonded interactions are calculated on GPU(s). - * \param[in] useGpuForUpdate If coordinate update and constraint solving is performed on + * \param[in] useGpuForBonded Whether bonded interactions are calculated on GPU(s). + * \param[in] useGpuForUpdate Whether coordinate update and constraint solving is performed on * GPU(s). - * \param[in] useGpuForBufferOps If buffer ops / reduction are calculated on GPU(s). - * \param[in] useGpuHaloExchange If GPU direct communication is used in halo exchange. - * \param[in] useGpuPmePpComm If GPu direct communication is used in PME-PP communication. + * \param[in] useGpuForBufferOps Whether buffer ops / reduction are calculated on GPU(s). + * \param[in] useGpuHaloExchange Whether GPU direct communication is used in halo exchange. + * \param[in] useGpuPmePpComm Whether GPU direct communication is used in PME-PP communication. + * \param[in] haveEwaldSurfaceContribution Whether there is an Ewald surface contribution * \returns Simulation lifetime constant workload description. */ -SimulationWorkload createSimulationWorkload(bool useGpuForNonbonded, - PmeRunMode pmeRunMode, - bool useGpuForBonded, - bool useGpuForUpdate, - bool useGpuForBufferOps, - bool useGpuHaloExchange, - bool useGpuPmePpComm); - +SimulationWorkload createSimulationWorkload(bool useGpuForNonbonded, + PmeRunMode pmeRunMode, + bool useGpuForBonded, + bool useGpuForUpdate, + bool useGpuForBufferOps, + bool useGpuHaloExchange, + bool useGpuPmePpComm, + bool haveEwaldSurfaceContribution); } // namespace gmx -- 2.22.0