From 1d69cda20e7800b5ac8659fa4037fa5adb67a194 Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Mon, 2 Dec 2019 14:15:08 +0100 Subject: [PATCH] Add check for coupled constraints for CUDA update The CUDA LINCS code has a limit on the number of coupled constraints. This is now checked during in the GPU update decision function. Change-Id: I5eee96b82f3f5196b64a5e815eae78d4ed367a80 --- src/gromacs/mdlib/lincs_cuda.cu | 21 +++++++++++++++++++ src/gromacs/mdlib/lincs_cuda.cuh | 7 +++++++ src/gromacs/mdlib/update_constrain_cuda.h | 8 +++++++ .../mdlib/update_constrain_cuda_impl.cpp | 5 +++++ .../mdlib/update_constrain_cuda_impl.cu | 5 +++++ .../mdlib/update_constrain_cuda_impl.h | 8 +++++++ src/gromacs/mdrun/runner.cpp | 3 +-- src/gromacs/taskassignment/decidegpuusage.cpp | 21 ++++++++++++++----- src/gromacs/taskassignment/decidegpuusage.h | 4 ++-- 9 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/gromacs/mdlib/lincs_cuda.cu b/src/gromacs/mdlib/lincs_cuda.cu index e1bf40d8d2..04d7b751b2 100644 --- a/src/gromacs/mdlib/lincs_cuda.cu +++ b/src/gromacs/mdlib/lincs_cuda.cu @@ -70,6 +70,7 @@ #include "gromacs/pbcutil/pbc.h" #include "gromacs/pbcutil/pbc_aiuc_cuda.cuh" #include "gromacs/topology/ifunc.h" +#include "gromacs/topology/topology.h" namespace gmx { @@ -681,6 +682,26 @@ static std::vector countNumCoupledConstraints(ArrayRef iatoms, return numCoupledConstraints; } +bool LincsCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop) +{ + for (const gmx_moltype_t& molType : mtop.moltype) + { + ArrayRef iatoms = molType.ilist[F_CONSTR].iatoms; + const auto atomsAdjacencyList = constructAtomsAdjacencyList(molType.atoms.nr, iatoms); + // Compute, how many constraints are coupled to each constraint + const auto numCoupledConstraints = countNumCoupledConstraints(iatoms, atomsAdjacencyList); + for (const int numCoupled : numCoupledConstraints) + { + if (numCoupled > c_threadsPerBlock) + { + return false; + } + } + } + + return true; +} + void LincsCuda::set(const t_idef& idef, const t_mdatoms& md) { int numAtoms = md.nr; diff --git a/src/gromacs/mdlib/lincs_cuda.cuh b/src/gromacs/mdlib/lincs_cuda.cuh index 99589f33cf..06c8bb40ab 100644 --- a/src/gromacs/mdlib/lincs_cuda.cuh +++ b/src/gromacs/mdlib/lincs_cuda.cuh @@ -166,6 +166,13 @@ public: */ void setPbc(const t_pbc* pbc); + /*! \brief + * Returns whether the maximum number of coupled constraints is supported + * by the CUDA LINCS code. + * + * \param[in] mtop The molecular topology + */ + static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop); private: //! CUDA stream diff --git a/src/gromacs/mdlib/update_constrain_cuda.h b/src/gromacs/mdlib/update_constrain_cuda.h index ee6ff74b07..5adca1c433 100644 --- a/src/gromacs/mdlib/update_constrain_cuda.h +++ b/src/gromacs/mdlib/update_constrain_cuda.h @@ -154,6 +154,14 @@ public: */ GpuEventSynchronizer* getCoordinatesReadySync(); + /*! \brief + * Returns whether the maximum number of coupled constraints is supported + * by the CUDA LINCS code. + * + * \param[in] mtop The molecular topology + */ + static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop); + private: class Impl; gmx::PrivateImplPointer impl_; diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.cpp b/src/gromacs/mdlib/update_constrain_cuda_impl.cpp index 82c19e7359..adbf2f5ba5 100644 --- a/src/gromacs/mdlib/update_constrain_cuda_impl.cpp +++ b/src/gromacs/mdlib/update_constrain_cuda_impl.cpp @@ -112,6 +112,11 @@ GpuEventSynchronizer* UpdateConstrainCuda::getCoordinatesReadySync() return nullptr; } +bool UpdateConstrainCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& /* mtop */) +{ + return false; +} + } // namespace gmx #endif /* GMX_GPU != GMX_GPU_CUDA */ diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.cu b/src/gromacs/mdlib/update_constrain_cuda_impl.cu index add995f26a..c11a74ad81 100644 --- a/src/gromacs/mdlib/update_constrain_cuda_impl.cu +++ b/src/gromacs/mdlib/update_constrain_cuda_impl.cu @@ -282,4 +282,9 @@ GpuEventSynchronizer* UpdateConstrainCuda::getCoordinatesReadySync() return impl_->getCoordinatesReadySync(); } +bool UpdateConstrainCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop) +{ + return LincsCuda::isNumCoupledConstraintsSupported(mtop); +} + } // namespace gmx diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.h b/src/gromacs/mdlib/update_constrain_cuda_impl.h index 62ff01b19c..68fed99c6b 100644 --- a/src/gromacs/mdlib/update_constrain_cuda_impl.h +++ b/src/gromacs/mdlib/update_constrain_cuda_impl.h @@ -153,6 +153,14 @@ public: */ GpuEventSynchronizer* getCoordinatesReadySync(); + /*! \brief + * Returns whether the maximum number of coupled constraints is supported + * by the CUDA LINCS code. + * + * \param[in] mtop The molecular topology + */ + static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop); + private: //! CUDA stream CommandStream commandStream_ = nullptr; diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index d68a5fec54..fd9700e453 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -909,8 +909,7 @@ int Mdrunner::mdrunner() { useGpuForUpdate = decideWhetherToUseGpuForUpdate( devFlags.forceGpuUpdateDefaultOn, useDomainDecomposition, useGpuForPme, - useGpuForNonbonded, updateTarget, gpusWereDetected, *inputrec, - gmx_mtop_interaction_count(mtop, IF_VSITE) > 0, doEssentialDynamics, + useGpuForNonbonded, updateTarget, gpusWereDetected, *inputrec, mtop, doEssentialDynamics, gmx_mtop_ftype_count(mtop, F_ORIRES) > 0, replExParams.exchangeInterval > 0); } GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR diff --git a/src/gromacs/taskassignment/decidegpuusage.cpp b/src/gromacs/taskassignment/decidegpuusage.cpp index b4ab67075d..0cd35168ea 100644 --- a/src/gromacs/taskassignment/decidegpuusage.cpp +++ b/src/gromacs/taskassignment/decidegpuusage.cpp @@ -57,11 +57,13 @@ #include "gromacs/hardware/hardwaretopology.h" #include "gromacs/hardware/hw_info.h" #include "gromacs/mdlib/gmx_omp_nthreads.h" +#include "gromacs/mdlib/update_constrain_cuda.h" #include "gromacs/mdtypes/commrec.h" #include "gromacs/mdtypes/inputrec.h" #include "gromacs/mdtypes/md_enums.h" #include "gromacs/mdtypes/mdrunoptions.h" #include "gromacs/taskassignment/taskassignment.h" +#include "gromacs/topology/mtop_util.h" #include "gromacs/topology/topology.h" #include "gromacs/utility/baseversion.h" #include "gromacs/utility/exceptions.h" @@ -493,7 +495,7 @@ bool decideWhetherToUseGpuForUpdate(const bool forceGpuUpdateDefaultOn, const TaskTarget updateTarget, const bool gpusWereDetected, const t_inputrec& inputrec, - const bool haveVSites, + const gmx_mtop_t& mtop, const bool useEssentialDynamics, const bool doOrientationRestraints, const bool useReplicaExchange) @@ -543,7 +545,7 @@ bool decideWhetherToUseGpuForUpdate(const bool forceGpuUpdateDefaultOn, // The graph is needed, but not supported errorMessage += "Ewald surface correction is not supported.\n"; } - if (haveVSites) + if (gmx_mtop_interaction_count(mtop, IF_VSITE) > 0) { errorMessage += "Virtual sites are not supported.\n"; } @@ -575,9 +577,18 @@ bool decideWhetherToUseGpuForUpdate(const bool forceGpuUpdateDefaultOn, errorMessage += "Swapping the coordinates is not supported.\n"; } - // \todo Check for coupled constraint block size restriction needs to be added - // when update auto chooses GPU in some cases. Currently exceeding the restriction - // triggers a fatal error during LINCS setup. + // TODO: F_CONSTRNC is only unsupported, because isNumCoupledConstraintsSupported() + // does not support it, the actual CUDA LINCS code does support it + if (gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0) + { + errorMessage += "Non-connecting constraints are not supported"; + } + if (!UpdateConstrainCuda::isNumCoupledConstraintsSupported(mtop)) + { + errorMessage += + "The number of coupled constraints is higher than supported in the CUDA LINCS " + "code.\n"; + } if (!errorMessage.empty()) { diff --git a/src/gromacs/taskassignment/decidegpuusage.h b/src/gromacs/taskassignment/decidegpuusage.h index d564043ca0..5bbc1b231c 100644 --- a/src/gromacs/taskassignment/decidegpuusage.h +++ b/src/gromacs/taskassignment/decidegpuusage.h @@ -238,7 +238,7 @@ bool decideWhetherToUseGpusForBonded(bool useGpuForNonbonded, * \param[in] updateTarget User choice for running simulation on GPU. * \param[in] gpusWereDetected Whether compatible GPUs were detected on any node. * \param[in] inputrec The user input. - * \param[in] haveVSites If there are virtual sites in the system. + * \param[in] mtop The global topology. * \param[in] useEssentialDynamics If essential dynamics is active. * \param[in] doOrientationRestraints If orientation restraints are enabled. * \param[in] useReplicaExchange If this is a REMD simulation. @@ -254,7 +254,7 @@ bool decideWhetherToUseGpuForUpdate(bool forceGpuUpdateDefaultOn, TaskTarget updateTarget, bool gpusWereDetected, const t_inputrec& inputrec, - bool haveVSites, + const gmx_mtop_t& mtop, bool useEssentialDynamics, bool doOrientationRestraints, bool useReplicaExchange); -- 2.22.0