Add check for coupled constraints for CUDA update

The CUDA LINCS code has a limit on the number of coupled constraints.
This is now checked during in the GPU update decision function.

Change-Id: I5eee96b82f3f5196b64a5e815eae78d4ed367a80

diff --git a/src/gromacs/mdlib/lincs_cuda.cu b/src/gromacs/mdlib/lincs_cuda.cu
index e1bf40d8d2157f9c62e50868a6613c3d8c331e18..04d7b751b24aa9e7aaef162ef68eaf06e25db085 100644 (file)
--- a/src/gromacs/mdlib/lincs_cuda.cu
+++ b/src/gromacs/mdlib/lincs_cuda.cu
@@ -70,6 +70,7 @@
 #include "gromacs/pbcutil/pbc.h"
 #include "gromacs/pbcutil/pbc_aiuc_cuda.cuh"
 #include "gromacs/topology/ifunc.h"
+#include "gromacs/topology/topology.h"
 
 namespace gmx
 {
@@ -681,6 +682,26 @@ static std::vector<int> countNumCoupledConstraints(ArrayRef<const int> iatoms,
     return numCoupledConstraints;
 }
 
+bool LincsCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop)
+{
+    for (const gmx_moltype_t& molType : mtop.moltype)
+    {
+        ArrayRef<const int> iatoms    = molType.ilist[F_CONSTR].iatoms;
+        const auto atomsAdjacencyList = constructAtomsAdjacencyList(molType.atoms.nr, iatoms);
+        // Compute, how many constraints are coupled to each constraint
+        const auto numCoupledConstraints = countNumCoupledConstraints(iatoms, atomsAdjacencyList);
+        for (const int numCoupled : numCoupledConstraints)
+        {
+            if (numCoupled > c_threadsPerBlock)
+            {
+                return false;
+            }
+        }
+    }
+
+    return true;
+}
+
 void LincsCuda::set(const t_idef& idef, const t_mdatoms& md)
 {
     int numAtoms = md.nr;

diff --git a/src/gromacs/mdlib/lincs_cuda.cuh b/src/gromacs/mdlib/lincs_cuda.cuh
index 99589f33cf9ec843c5fe315caa8328240ab0cd1c..06c8bb40ab0eb4abf0837c90227dd93d69a65e97 100644 (file)
--- a/src/gromacs/mdlib/lincs_cuda.cuh
+++ b/src/gromacs/mdlib/lincs_cuda.cuh
@@ -166,6 +166,13 @@ public:
      */
     void setPbc(const t_pbc* pbc);
 
+    /*! \brief
+     * Returns whether the maximum number of coupled constraints is supported
+     * by the CUDA LINCS code.
+     *
+     * \param[in] mtop The molecular topology
+     */
+    static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop);
 
 private:
     //! CUDA stream

diff --git a/src/gromacs/mdlib/update_constrain_cuda.h b/src/gromacs/mdlib/update_constrain_cuda.h
index ee6ff74b07b8c586f8936306aa00336c63e793df..5adca1c4339957059184cd00066d853f34071a7b 100644 (file)
--- a/src/gromacs/mdlib/update_constrain_cuda.h
+++ b/src/gromacs/mdlib/update_constrain_cuda.h
@@ -154,6 +154,14 @@ public:
      */
     GpuEventSynchronizer* getCoordinatesReadySync();
 
+    /*! \brief
+     * Returns whether the maximum number of coupled constraints is supported
+     * by the CUDA LINCS code.
+     *
+     * \param[in] mtop The molecular topology
+     */
+    static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop);
+
 private:
     class Impl;
     gmx::PrivateImplPointer<Impl> impl_;

diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.cpp b/src/gromacs/mdlib/update_constrain_cuda_impl.cpp
index 82c19e735903feabd6e5b3673a2deacbf1e134b5..adbf2f5ba5ce60293f1276ebe8f3730e8ca5d3ac 100644 (file)
--- a/src/gromacs/mdlib/update_constrain_cuda_impl.cpp
+++ b/src/gromacs/mdlib/update_constrain_cuda_impl.cpp
@@ -112,6 +112,11 @@ GpuEventSynchronizer* UpdateConstrainCuda::getCoordinatesReadySync()
     return nullptr;
 }
 
+bool UpdateConstrainCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& /* mtop */)
+{
+    return false;
+}
+
 } // namespace gmx
 
 #endif /* GMX_GPU != GMX_GPU_CUDA */

diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.cu b/src/gromacs/mdlib/update_constrain_cuda_impl.cu
index add995f26a0322b281321bf1608557f319e417a4..c11a74ad819ba0a58154f6f513723184bf75ee01 100644 (file)
--- a/src/gromacs/mdlib/update_constrain_cuda_impl.cu
+++ b/src/gromacs/mdlib/update_constrain_cuda_impl.cu
@@ -282,4 +282,9 @@ GpuEventSynchronizer* UpdateConstrainCuda::getCoordinatesReadySync()
     return impl_->getCoordinatesReadySync();
 }
 
+bool UpdateConstrainCuda::isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop)
+{
+    return LincsCuda::isNumCoupledConstraintsSupported(mtop);
+}
+
 } // namespace gmx

diff --git a/src/gromacs/mdlib/update_constrain_cuda_impl.h b/src/gromacs/mdlib/update_constrain_cuda_impl.h
index 62ff01b19c6105fc752c4f7e62c741bd072185cd..68fed99c6b3dae24f48e190f3b44c29d5aa6a9df 100644 (file)
--- a/src/gromacs/mdlib/update_constrain_cuda_impl.h
+++ b/src/gromacs/mdlib/update_constrain_cuda_impl.h
@@ -153,6 +153,14 @@ public:
      */
     GpuEventSynchronizer* getCoordinatesReadySync();
 
+    /*! \brief
+     * Returns whether the maximum number of coupled constraints is supported
+     * by the CUDA LINCS code.
+     *
+     * \param[in] mtop The molecular topology
+     */
+    static bool isNumCoupledConstraintsSupported(const gmx_mtop_t& mtop);
+
 private:
     //! CUDA stream
     CommandStream commandStream_ = nullptr;

diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp
index d68a5fec5414746cd2e373289dfcfc7db653f9f3..fd9700e4536e51dadeca872f0c79849edfe59eaf 100644 (file)
--- a/src/gromacs/mdrun/runner.cpp
+++ b/src/gromacs/mdrun/runner.cpp
@@ -909,8 +909,7 @@ int Mdrunner::mdrunner()
     {
         useGpuForUpdate = decideWhetherToUseGpuForUpdate(
                 devFlags.forceGpuUpdateDefaultOn, useDomainDecomposition, useGpuForPme,
-                useGpuForNonbonded, updateTarget, gpusWereDetected, *inputrec,
-                gmx_mtop_interaction_count(mtop, IF_VSITE) > 0, doEssentialDynamics,
+                useGpuForNonbonded, updateTarget, gpusWereDetected, *inputrec, mtop, doEssentialDynamics,
                 gmx_mtop_ftype_count(mtop, F_ORIRES) > 0, replExParams.exchangeInterval > 0);
     }
     GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR

diff --git a/src/gromacs/taskassignment/decidegpuusage.cpp b/src/gromacs/taskassignment/decidegpuusage.cpp
index b4ab67075d1383c3ee6ea69ad28e91b00570ca5d..0cd35168ea4d06ead164088b725bce17c6a22de4 100644 (file)
--- a/src/gromacs/taskassignment/decidegpuusage.cpp
+++ b/src/gromacs/taskassignment/decidegpuusage.cpp
@@ -57,11 +57,13 @@
 #include "gromacs/hardware/hardwaretopology.h"
 #include "gromacs/hardware/hw_info.h"
 #include "gromacs/mdlib/gmx_omp_nthreads.h"
+#include "gromacs/mdlib/update_constrain_cuda.h"
 #include "gromacs/mdtypes/commrec.h"
 #include "gromacs/mdtypes/inputrec.h"
 #include "gromacs/mdtypes/md_enums.h"
 #include "gromacs/mdtypes/mdrunoptions.h"
 #include "gromacs/taskassignment/taskassignment.h"
+#include "gromacs/topology/mtop_util.h"
 #include "gromacs/topology/topology.h"
 #include "gromacs/utility/baseversion.h"
 #include "gromacs/utility/exceptions.h"
@@ -493,7 +495,7 @@ bool decideWhetherToUseGpuForUpdate(const bool        forceGpuUpdateDefaultOn,
                                     const TaskTarget  updateTarget,
                                     const bool        gpusWereDetected,
                                     const t_inputrec& inputrec,
-                                    const bool        haveVSites,
+                                    const gmx_mtop_t& mtop,
                                     const bool        useEssentialDynamics,
                                     const bool        doOrientationRestraints,
                                     const bool        useReplicaExchange)
@@ -543,7 +545,7 @@ bool decideWhetherToUseGpuForUpdate(const bool        forceGpuUpdateDefaultOn,
         // The graph is needed, but not supported
         errorMessage += "Ewald surface correction is not supported.\n";
     }
-    if (haveVSites)
+    if (gmx_mtop_interaction_count(mtop, IF_VSITE) > 0)
     {
         errorMessage += "Virtual sites are not supported.\n";
     }
@@ -575,9 +577,18 @@ bool decideWhetherToUseGpuForUpdate(const bool        forceGpuUpdateDefaultOn,
         errorMessage += "Swapping the coordinates is not supported.\n";
     }
 
-    // \todo Check for coupled constraint block size restriction needs to be added
-    //       when update auto chooses GPU in some cases. Currently exceeding the restriction
-    //       triggers a fatal error during LINCS setup.
+    // TODO: F_CONSTRNC is only unsupported, because isNumCoupledConstraintsSupported()
+    // does not support it, the actual CUDA LINCS code does support it
+    if (gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0)
+    {
+        errorMessage += "Non-connecting constraints are not supported";
+    }
+    if (!UpdateConstrainCuda::isNumCoupledConstraintsSupported(mtop))
+    {
+        errorMessage +=
+                "The number of coupled constraints is higher than supported in the CUDA LINCS "
+                "code.\n";
+    }
 
     if (!errorMessage.empty())
     {

diff --git a/src/gromacs/taskassignment/decidegpuusage.h b/src/gromacs/taskassignment/decidegpuusage.h
index d564043ca00d227be02b69cd0062e06728e3eec8..5bbc1b231c97081b8a1c27a0d5d08c932f640162 100644 (file)
--- a/src/gromacs/taskassignment/decidegpuusage.h
+++ b/src/gromacs/taskassignment/decidegpuusage.h
@@ -238,7 +238,7 @@ bool decideWhetherToUseGpusForBonded(bool       useGpuForNonbonded,
  * \param[in]  updateTarget              User choice for running simulation on GPU.
  * \param[in]  gpusWereDetected          Whether compatible GPUs were detected on any node.
  * \param[in]  inputrec                  The user input.
- * \param[in]  haveVSites                If there are virtual sites in the system.
+ * \param[in]  mtop                      The global topology.
  * \param[in]  useEssentialDynamics      If essential dynamics is active.
  * \param[in]  doOrientationRestraints   If orientation restraints are enabled.
  * \param[in]  useReplicaExchange        If this is a REMD simulation.
@@ -254,7 +254,7 @@ bool decideWhetherToUseGpuForUpdate(bool              forceGpuUpdateDefaultOn,
                                     TaskTarget        updateTarget,
                                     bool              gpusWereDetected,
                                     const t_inputrec& inputrec,
-                                    bool              haveVSites,
+                                    const gmx_mtop_t& mtop,
                                     bool              useEssentialDynamics,
                                     bool              doOrientationRestraints,
                                     bool              useReplicaExchange);