From 543017dfaebacab4720bce6b6b9d9d3b7c5a9917 Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Thu, 17 Sep 2020 14:13:57 +0200 Subject: [PATCH] Move bNonbonded flag from t_forcerec to SimulationWorkload The flag to store whether nonbonded calculations are requested to be skipped (set by env.var.) is move from t_forcerec to SimulationWorkload. A minor side-effect of this change is that the nbnxn module logs kernels choices when this option is set. --- src/gromacs/mdlib/forcerec.cpp | 12 ----- src/gromacs/mdlib/sim_util.cpp | 25 +++++------ src/gromacs/mdrun/runner.cpp | 16 ++++++- src/gromacs/mdtypes/forcerec.h | 3 +- src/gromacs/mdtypes/simulation_workload.h | 2 + src/gromacs/nbnxm/nbnxm_setup.cpp | 45 ++++++++----------- .../decidesimulationworkload.cpp | 12 ++--- .../taskassignment/decidesimulationworkload.h | 4 +- 8 files changed, 57 insertions(+), 62 deletions(-) diff --git a/src/gromacs/mdlib/forcerec.cpp b/src/gromacs/mdlib/forcerec.cpp index 1bd9658392..8dc12a3be3 100644 --- a/src/gromacs/mdlib/forcerec.cpp +++ b/src/gromacs/mdlib/forcerec.cpp @@ -1006,18 +1006,6 @@ void init_forcerec(FILE* fp, /* Free energy */ fr->efep = ir->efep; - fr->bNonbonded = TRUE; - if (getenv("GMX_NO_NONBONDED") != nullptr) - { - /* turn off non-bonded calculations */ - fr->bNonbonded = FALSE; - GMX_LOG(mdlog.warning) - .asParagraph() - .appendText( - "Found environment variable GMX_NO_NONBONDED.\n" - "Disabling nonbonded calculations."); - } - if ((getenv("GMX_DISABLE_SIMD_KERNELS") != nullptr) || (getenv("GMX_NOOPTIMIZEDKERNELS") != nullptr)) { fr->use_simd_kernels = FALSE; diff --git a/src/gromacs/mdlib/sim_util.cpp b/src/gromacs/mdlib/sim_util.cpp index 72160eda1a..7d1109889b 100644 --- a/src/gromacs/mdlib/sim_util.cpp +++ b/src/gromacs/mdlib/sim_util.cpp @@ -881,27 +881,26 @@ static DomainLifetimeWorkload setupDomainLifetimeWorkload(const t_inputrec& /*! \brief Set up force flag stuct from the force bitmask. * * \param[in] legacyFlags Force bitmask flags used to construct the new flags - * \param[in] isNonbondedOn Global override, if false forces to turn off all nonbonded calculation. * \param[in] simulationWork Simulation workload description. * \param[in] rankHasPmeDuty If this rank computes PME. * * \returns New Stepworkload description. */ static StepWorkload setupStepWorkload(const int legacyFlags, - const bool isNonbondedOn, const SimulationWorkload& simulationWork, const bool rankHasPmeDuty) { StepWorkload flags; - flags.stateChanged = ((legacyFlags & GMX_FORCE_STATECHANGED) != 0); - flags.haveDynamicBox = ((legacyFlags & GMX_FORCE_DYNAMICBOX) != 0); - flags.doNeighborSearch = ((legacyFlags & GMX_FORCE_NS) != 0); - flags.computeVirial = ((legacyFlags & GMX_FORCE_VIRIAL) != 0); - flags.computeEnergy = ((legacyFlags & GMX_FORCE_ENERGY) != 0); - flags.computeForces = ((legacyFlags & GMX_FORCE_FORCES) != 0); - flags.computeListedForces = ((legacyFlags & GMX_FORCE_LISTED) != 0); - flags.computeNonbondedForces = ((legacyFlags & GMX_FORCE_NONBONDED) != 0) && isNonbondedOn; - flags.computeDhdl = ((legacyFlags & GMX_FORCE_DHDL) != 0); + flags.stateChanged = ((legacyFlags & GMX_FORCE_STATECHANGED) != 0); + flags.haveDynamicBox = ((legacyFlags & GMX_FORCE_DYNAMICBOX) != 0); + flags.doNeighborSearch = ((legacyFlags & GMX_FORCE_NS) != 0); + flags.computeVirial = ((legacyFlags & GMX_FORCE_VIRIAL) != 0); + flags.computeEnergy = ((legacyFlags & GMX_FORCE_ENERGY) != 0); + flags.computeForces = ((legacyFlags & GMX_FORCE_FORCES) != 0); + flags.computeListedForces = ((legacyFlags & GMX_FORCE_LISTED) != 0); + flags.computeNonbondedForces = + ((legacyFlags & GMX_FORCE_NONBONDED) != 0) && simulationWork.computeNonbonded; + flags.computeDhdl = ((legacyFlags & GMX_FORCE_DHDL) != 0); if (simulationWork.useGpuBufferOps) { @@ -1053,8 +1052,8 @@ void do_force(FILE* fplog, const SimulationWorkload& simulationWork = runScheduleWork->simulationWork; - runScheduleWork->stepWork = setupStepWorkload(legacyFlags, fr->bNonbonded, simulationWork, - thisRankHasDuty(cr, DUTY_PME)); + runScheduleWork->stepWork = + setupStepWorkload(legacyFlags, simulationWork, thisRankHasDuty(cr, DUTY_PME)); const StepWorkload& stepWork = runScheduleWork->stepWork; diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index 781ccea41c..e0802407fc 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -1247,10 +1247,22 @@ int Mdrunner::mdrunner() const bool printHostName = (cr->nnodes > 1); gpuTaskAssignments.reportGpuUsage(mdlog, printHostName, useGpuForBonded, pmeRunMode, useGpuForUpdate); + const bool disableNonbondedCalculation = (getenv("GMX_NO_NONBONDED") != nullptr); + if (disableNonbondedCalculation) + { + /* turn off non-bonded calculations */ + GMX_LOG(mdlog.warning) + .asParagraph() + .appendText( + "Found environment variable GMX_NO_NONBONDED.\n" + "Disabling nonbonded calculations."); + } + MdrunScheduleWorkload runScheduleWork; // Also populates the simulation constant workload description. - runScheduleWork.simulationWork = createSimulationWorkload( - *inputrec, devFlags, useGpuForNonbonded, pmeRunMode, useGpuForBonded, useGpuForUpdate); + runScheduleWork.simulationWork = + createSimulationWorkload(*inputrec, disableNonbondedCalculation, devFlags, + useGpuForNonbonded, pmeRunMode, useGpuForBonded, useGpuForUpdate); std::unique_ptr deviceStreamManager = nullptr; diff --git a/src/gromacs/mdtypes/forcerec.h b/src/gromacs/mdtypes/forcerec.h index b48bd0910d..9077b4b35d 100644 --- a/src/gromacs/mdtypes/forcerec.h +++ b/src/gromacs/mdtypes/forcerec.h @@ -242,8 +242,7 @@ struct t_forcerec std::unique_ptr wholeMoleculeTransform; - int cutoff_scheme = 0; /* group- or Verlet-style cutoff */ - gmx_bool bNonbonded = FALSE; /* true if nonbonded calculations are *not* turned off */ + int cutoff_scheme = 0; /* group- or Verlet-style cutoff */ /* The Nbnxm Verlet non-bonded machinery */ std::unique_ptr nbv; diff --git a/src/gromacs/mdtypes/simulation_workload.h b/src/gromacs/mdtypes/simulation_workload.h index 8f75ba7436..0763e6f125 100644 --- a/src/gromacs/mdtypes/simulation_workload.h +++ b/src/gromacs/mdtypes/simulation_workload.h @@ -140,6 +140,8 @@ public: class SimulationWorkload { public: + //! Whether to compute nonbonded pair interactions + bool computeNonbonded = false; //! Whether total dipole needs to be computed bool computeMuTot = false; //! If we have calculation of short range nonbondeds on CPU diff --git a/src/gromacs/nbnxm/nbnxm_setup.cpp b/src/gromacs/nbnxm/nbnxm_setup.cpp index bcb0b7d926..74086df3de 100644 --- a/src/gromacs/nbnxm/nbnxm_setup.cpp +++ b/src/gromacs/nbnxm/nbnxm_setup.cpp @@ -238,8 +238,7 @@ static KernelSetup pick_nbnxn_kernel(const gmx::MDLogger& mdlog, gmx_bool use_simd_kernels, const gmx_hw_info_t& hardwareInfo, const NonbondedResource& nonbondedResource, - const t_inputrec* ir, - gmx_bool bDoNonbonded) + const t_inputrec* ir) { KernelSetup kernelSetup; @@ -248,12 +247,7 @@ static KernelSetup pick_nbnxn_kernel(const gmx::MDLogger& mdlog, kernelSetup.kernelType = KernelType::Cpu8x8x8_PlainC; kernelSetup.ewaldExclusionType = EwaldExclusionType::DecidedByGpuModule; - if (bDoNonbonded) - { - GMX_LOG(mdlog.warning) - .asParagraph() - .appendText("Emulating a GPU run on the CPU (slow)"); - } + GMX_LOG(mdlog.warning).asParagraph().appendText("Emulating a GPU run on the CPU (slow)"); } else if (nonbondedResource == NonbondedResource::Gpu) { @@ -273,25 +267,22 @@ static KernelSetup pick_nbnxn_kernel(const gmx::MDLogger& mdlog, } } - if (bDoNonbonded) + GMX_LOG(mdlog.info) + .asParagraph() + .appendTextFormatted("Using %s %dx%d nonbonded short-range kernels", + lookup_kernel_name(kernelSetup.kernelType), + IClusterSizePerKernelType[kernelSetup.kernelType], + JClusterSizePerKernelType[kernelSetup.kernelType]); + + if (KernelType::Cpu4x4_PlainC == kernelSetup.kernelType + || KernelType::Cpu8x8x8_PlainC == kernelSetup.kernelType) { - GMX_LOG(mdlog.info) + GMX_LOG(mdlog.warning) .asParagraph() - .appendTextFormatted("Using %s %dx%d nonbonded short-range kernels", - lookup_kernel_name(kernelSetup.kernelType), - IClusterSizePerKernelType[kernelSetup.kernelType], - JClusterSizePerKernelType[kernelSetup.kernelType]); - - if (KernelType::Cpu4x4_PlainC == kernelSetup.kernelType - || KernelType::Cpu8x8x8_PlainC == kernelSetup.kernelType) - { - GMX_LOG(mdlog.warning) - .asParagraph() - .appendTextFormatted( - "WARNING: Using the slow %s kernels. This should\n" - "not happen during routine usage on supported platforms.", - lookup_kernel_name(kernelSetup.kernelType)); - } + .appendTextFormatted( + "WARNING: Using the slow %s kernels. This should\n" + "not happen during routine usage on supported platforms.", + lookup_kernel_name(kernelSetup.kernelType)); } GMX_RELEASE_ASSERT(kernelSetup.kernelType != KernelType::NotSet @@ -387,8 +378,8 @@ std::unique_ptr init_nb_verlet(const gmx::MDLogger& mdlog, nonbondedResource = NonbondedResource::Cpu; } - Nbnxm::KernelSetup kernelSetup = pick_nbnxn_kernel(mdlog, fr->use_simd_kernels, hardwareInfo, - nonbondedResource, ir, fr->bNonbonded); + Nbnxm::KernelSetup kernelSetup = + pick_nbnxn_kernel(mdlog, fr->use_simd_kernels, hardwareInfo, nonbondedResource, ir); const bool haveMultipleDomains = havePPDomainDecomposition(cr); diff --git a/src/gromacs/taskassignment/decidesimulationworkload.cpp b/src/gromacs/taskassignment/decidesimulationworkload.cpp index ad9d3050ed..60abedf026 100644 --- a/src/gromacs/taskassignment/decidesimulationworkload.cpp +++ b/src/gromacs/taskassignment/decidesimulationworkload.cpp @@ -51,7 +51,8 @@ namespace gmx { -SimulationWorkload createSimulationWorkload(const t_inputrec& inputrec, +SimulationWorkload createSimulationWorkload(const t_inputrec& inputrec, + const bool disableNonbondedCalculation, const DevelopmentFeatureFlags& devFlags, bool useGpuForNonbonded, PmeRunMode pmeRunMode, @@ -59,10 +60,11 @@ SimulationWorkload createSimulationWorkload(const t_inputrec& input bool useGpuForUpdate) { SimulationWorkload simulationWorkload; - simulationWorkload.computeMuTot = inputrecNeedMutot(&inputrec); - simulationWorkload.useCpuNonbonded = !useGpuForNonbonded; - simulationWorkload.useGpuNonbonded = useGpuForNonbonded; - simulationWorkload.useCpuPme = (pmeRunMode == PmeRunMode::CPU); + simulationWorkload.computeNonbonded = !disableNonbondedCalculation; + simulationWorkload.computeMuTot = inputrecNeedMutot(&inputrec); + simulationWorkload.useCpuNonbonded = !useGpuForNonbonded; + simulationWorkload.useGpuNonbonded = useGpuForNonbonded; + simulationWorkload.useCpuPme = (pmeRunMode == PmeRunMode::CPU); simulationWorkload.useGpuPme = (pmeRunMode == PmeRunMode::GPU || pmeRunMode == PmeRunMode::Mixed); simulationWorkload.useGpuPmeFft = (pmeRunMode == PmeRunMode::Mixed); simulationWorkload.useGpuBonded = useGpuForBonded; diff --git a/src/gromacs/taskassignment/decidesimulationworkload.h b/src/gromacs/taskassignment/decidesimulationworkload.h index 7dd87e6073..5cde42e10c 100644 --- a/src/gromacs/taskassignment/decidesimulationworkload.h +++ b/src/gromacs/taskassignment/decidesimulationworkload.h @@ -59,6 +59,7 @@ struct DevelopmentFeatureFlags; * task on GPUs. * * \param[in] inputrec The input record + * \param[in] disableNonbondedCalculation Disable calculation of nonbonded forces * \param[in] devFlags The development feature flags * \param[in] useGpuForNonbonded Whether we have short-range nonbonded interactions * calculations on GPU(s). @@ -68,7 +69,8 @@ struct DevelopmentFeatureFlags; * GPU(s). * \returns Simulation lifetime constant workload description. */ -SimulationWorkload createSimulationWorkload(const t_inputrec& inputrec, +SimulationWorkload createSimulationWorkload(const t_inputrec& inputrec, + bool disableNonbondedCalculation, const DevelopmentFeatureFlags& devFlags, bool useGpuForNonbonded, PmeRunMode pmeRunMode, -- 2.22.0