pmeStream_ = &deviceStreamManager.stream(DeviceStreamType::Pme);
localStream_ = &deviceStreamManager.stream(DeviceStreamType::NonBondedLocal);
nonLocalStream_ = &deviceStreamManager.stream(DeviceStreamType::NonBondedNonLocal);
- // PME stream is used in OpenCL for H2D coordinate transfer
- updateStream_ = &deviceStreamManager.stream(
- GMX_GPU_OPENCL ? DeviceStreamType::Pme : DeviceStreamType::UpdateAndConstraints);
+ updateStream_ = &deviceStreamManager.stream(DeviceStreamType::UpdateAndConstraints);
// Map the atom locality to the stream that will be used for coordinates,
// velocities and forces transfers. Same streams are used for H2D and D2H copies.
nonLocalStream_ = nullptr;
updateStream_ = nullptr;
+ isPmeOnly_ = true;
// Only local/all coordinates are allowed to be copied in PME-only rank/ PME tests.
// This it temporary measure to make it safe to use this class in those cases.
reallocateDeviceBuffer(&d_f_, numAtomsAll_, &d_fSize_, &d_fCapacity_, deviceContext_);
// Clearing of the forces can be done in local stream since the nonlocal stream cannot reach
- // the force accumulation stage before syncing with the local stream. Only done in CUDA,
- // since the force buffer ops are not implemented in OpenCL.
- if (GMX_GPU_CUDA && d_fCapacity_ != d_fOldCapacity)
+ // the force accumulation stage before syncing with the local stream. Only done in CUDA and
+ // SYCL, since the force buffer ops are not implemented in OpenCL.
+ static constexpr bool sc_haveGpuFBufferOps = ((GMX_GPU_CUDA != 0) || (GMX_GPU_SYCL != 0));
+ if (sc_haveGpuFBufferOps && d_fCapacity_ != d_fOldCapacity)
{
clearDeviceBufferAsync(&d_f_, 0, d_fCapacity_, *localStream_);
}
copyToDevice(d_x_, h_x, d_xSize_, atomLocality, *deviceStream);
- // markEvent is skipped in OpenCL as:
- // - it's not needed, copy is done in the same stream as the only consumer task (PME)
- // - we don't consume the events in OpenCL which is not allowed by GpuEventSynchronizer (would leak memory).
- // TODO: remove this by adding an event-mark free flavor of this function
- if (GMX_GPU_CUDA)
+ // marking is skipped on the PME-rank mode as everything is on the same stream
+ if (!isPmeOnly_)
{
xReadyOnDevice_[atomLocality].markEvent(*deviceStream);
}
// and this is not a neighbor search step, then the consumer needs to wait for the update
// to complete. Otherwise, the coordinates are copied from the host and we need to wait for
// the copy event. Non-local coordinates are provided by the GPU halo exchange (if active), otherwise by H2D copy.
- //
- // In OpenCL no events are used as coordinate sync is not necessary
- if (GMX_GPU_OPENCL)
- {
- return nullptr;
- }
+
if (atomLocality == AtomLocality::NonLocal && stepWork.useGpuXHalo)
{
GMX_ASSERT(gpuCoordinateHaloLaunched != nullptr,
}
else
{
+ if (stepWork.doNeighborSearch && xUpdatedOnDeviceEvent_)
+ {
+ /* On search steps, we do not consume the result of the GPU update
+ * but rather that of a H2D transfer. So, we reset the event triggered after
+ * update to avoid leaving it unconsumed.
+ * Unfortunately, we don't always have the event marked either (e.g., on the
+ * first step) so we just reset it here.
+ * See Issue #3988. */
+ xUpdatedOnDeviceEvent_->reset();
+ }
return &xReadyOnDevice_[atomLocality];
}
}
wallcycle_stop(wcycle_, WallCycleCounter::WaitGpuStatePropagatorData);
}
+void StatePropagatorDataGpu::Impl::consumeCoordinatesCopiedToDeviceEvent(AtomLocality atomLocality)
+{
+ GMX_ASSERT(atomLocality < AtomLocality::Count, "Wrong atom locality.");
+ xReadyOnDevice_[atomLocality].consume();
+}
+
+void StatePropagatorDataGpu::Impl::resetCoordinatesCopiedToDeviceEvent(AtomLocality atomLocality)
+{
+ GMX_ASSERT(atomLocality < AtomLocality::Count, "Wrong atom locality.");
+ xReadyOnDevice_[atomLocality].reset();
+}
+
void StatePropagatorDataGpu::Impl::setXUpdatedOnDeviceEvent(GpuEventSynchronizer* xUpdatedOnDeviceEvent)
{
GMX_ASSERT(xUpdatedOnDeviceEvent != nullptr, "The event synchronizer can not be nullptr.");
return &fReducedOnDevice_[atomLocality];
}
+void StatePropagatorDataGpu::Impl::consumeForcesReducedOnDeviceEvent(AtomLocality atomLocality)
+{
+ fReducedOnDevice_[atomLocality].consume();
+}
+
GpuEventSynchronizer* StatePropagatorDataGpu::Impl::fReadyOnDevice(AtomLocality atomLocality)
{
return &fReadyOnDevice_[atomLocality];
return impl_->waitCoordinatesCopiedToDevice(atomLocality);
}
+void StatePropagatorDataGpu::consumeCoordinatesCopiedToDeviceEvent(AtomLocality atomLocality)
+{
+ return impl_->consumeCoordinatesCopiedToDeviceEvent(atomLocality);
+}
+
+void StatePropagatorDataGpu::resetCoordinatesCopiedToDeviceEvent(AtomLocality atomLocality)
+{
+ return impl_->resetCoordinatesCopiedToDeviceEvent(atomLocality);
+}
+
void StatePropagatorDataGpu::setXUpdatedOnDeviceEvent(GpuEventSynchronizer* xUpdatedOnDeviceEvent)
{
impl_->setXUpdatedOnDeviceEvent(xUpdatedOnDeviceEvent);
return impl_->fReducedOnDevice(atomLocality);
}
+void StatePropagatorDataGpu::consumeForcesReducedOnDeviceEvent(AtomLocality atomLocality)
+{
+ impl_->consumeForcesReducedOnDeviceEvent(atomLocality);
+}
+
GpuEventSynchronizer* StatePropagatorDataGpu::fReadyOnDevice(AtomLocality atomLocality)
{
return impl_->fReadyOnDevice(atomLocality);