Rework GPU halo and state propagator streams and dependencies to get better overlap

[alexxy/gromacs.git] / src / gromacs / mdtypes / state_propagator_data_gpu_impl.h

diff --git a/src/gromacs/mdtypes/state_propagator_data_gpu_impl.h b/src/gromacs/mdtypes/state_propagator_data_gpu_impl.h
index 4353eecf4ecf278d25ef632b8230b8cd45b03715..dbe435d508ddd13c54aeec0eeb7c21d26cae7784 100644 (file)
--- a/src/gromacs/mdtypes/state_propagator_data_gpu_impl.h
+++ b/src/gromacs/mdtypes/state_propagator_data_gpu_impl.h
@@ -175,12 +175,14 @@ public:
      *  \param[in] atomLocality    Locality of the particles to wait for.
      *  \param[in] simulationWork  The simulation lifetime flags.
      *  \param[in] stepWork        The step lifetime flags.
+     *  \param[in] gpuCoordinateHaloLaunched Event recorded when GPU coordinate halo has been launched.
      *
      *  \returns  The event to synchronize the stream that consumes coordinates on device.
      */
     GpuEventSynchronizer* getCoordinatesReadyOnDeviceEvent(AtomLocality              atomLocality,
                                                            const SimulationWorkload& simulationWork,
-                                                           const StepWorkload&       stepWork);
+                                                           const StepWorkload&       stepWork,
+                                                           GpuEventSynchronizer* gpuCoordinateHaloLaunched = nullptr);
 
     /*! \brief Blocking wait until coordinates are copied to the device.
      *
@@ -196,12 +198,15 @@ public:
      */
     void setXUpdatedOnDeviceEvent(GpuEventSynchronizer* xUpdatedOnDeviceEvent);
 
-    /*! \brief Copy positions from the GPU memory.
+    /*! \brief Copy positions from the GPU memory, with an optional explicit dependency.
      *
      *  \param[in] h_x           Positions buffer in the host memory.
      *  \param[in] atomLocality  Locality of the particles to copy.
+     *  \param[in] dependency    Dependency event for this operation.
      */
-    void copyCoordinatesFromGpu(gmx::ArrayRef<gmx::RVec> h_x, AtomLocality atomLocality);
+    void copyCoordinatesFromGpu(gmx::ArrayRef<gmx::RVec> h_x,
+                                AtomLocality             atomLocality,
+                                GpuEventSynchronizer*    dependency = nullptr);
 
     /*! \brief Wait until coordinates are available on the host.
      *
@@ -253,8 +258,9 @@ public:
     /*! \brief Clear forces in the GPU memory.
      *
      *  \param[in] atomLocality  Locality of the particles to clear.
+     *  \param[in] dependency    Dependency event for this operation.
      */
-    void clearForcesOnGpu(AtomLocality atomLocality);
+    void clearForcesOnGpu(AtomLocality atomLocality, GpuEventSynchronizer* dependency);
 
     /*! \brief Get the event synchronizer for the forces ready on device.
      *
@@ -263,20 +269,27 @@ public:
      *  1. The forces are copied to the device (when GPU buffer ops are off)
      *  2. The forces are reduced on the device (GPU buffer ops are on)
      *
-     *  \todo Pass step workload instead of the useGpuFBufferOps boolean.
-     *
-     *  \param[in] atomLocality      Locality of the particles to wait for.
-     *  \param[in] useGpuFBufferOps  If the force buffer ops are offloaded to the GPU.
+     *  \param[in] stepWork        Step workload flags
+     *  \param[in] simulationWork  Simulation workload flags
      *
      *  \returns  The event to synchronize the stream that consumes forces on device.
      */
-    GpuEventSynchronizer* getForcesReadyOnDeviceEvent(AtomLocality atomLocality, bool useGpuFBufferOps);
+    GpuEventSynchronizer* getLocalForcesReadyOnDeviceEvent(StepWorkload       stepWork,
+                                                           SimulationWorkload simulationWork);
 
-    /*! \brief Getter for the event synchronizer for the forces are reduced on the GPU.
+    /*! \brief Getter for the event synchronizer for when forces are reduced on the GPU.
      *
-     *  \returns  The event to mark when forces are reduced on the GPU.
+     *  \param[in] atomLocality      Locality of the particles to wait for.
+     *  \returns                     The event to mark when forces are reduced on the GPU.
+     */
+    GpuEventSynchronizer* fReducedOnDevice(AtomLocality atomLocality);
+
+    /*! \brief Getter for the event synchronizer for the forces are ready for GPU update.
+     *
+     *  \param[in] atomLocality      Locality of the particles to wait for.
+     *  \returns                     The event to mark when forces are ready for GPU update.
      */
-    GpuEventSynchronizer* fReducedOnDevice();
+    GpuEventSynchronizer* fReadyOnDevice(AtomLocality atomLocality);
 
     /*! \brief Copy forces from the GPU memory.
      *
@@ -327,6 +340,9 @@ private:
     EnumerationArray<AtomLocality, const DeviceStream*> vCopyStreams_ = { { nullptr } };
     // Streams to use for forces H2D and D2H copies (one event for each atom locality)
     EnumerationArray<AtomLocality, const DeviceStream*> fCopyStreams_ = { { nullptr } };
+    // Streams internal to this module
+    std::unique_ptr<DeviceStream> copyInStream_;
+    std::unique_ptr<DeviceStream> memsetStream_;
 
     /*! \brief An array of events that indicate H2D copy is complete (one event for each atom locality)
      *
@@ -343,8 +359,8 @@ private:
 
     //! An array of events that indicate H2D copy of forces is complete (one event for each atom locality)
     EnumerationArray<AtomLocality, GpuEventSynchronizer> fReadyOnDevice_;
-    //! An event that the forces were reduced on the GPU
-    GpuEventSynchronizer fReducedOnDevice_;
+    //! An array of events that indicate the forces were reduced on the GPU (one event for each atom locality)
+    EnumerationArray<AtomLocality, GpuEventSynchronizer> fReducedOnDevice_;
     //! An array of events that indicate D2H copy of forces is complete (one event for each atom locality)
     EnumerationArray<AtomLocality, GpuEventSynchronizer> fReadyOnHost_;