From 624e3341d38327f8640de8cacc24a9fe3a221233 Mon Sep 17 00:00:00 2001 From: Andrey Alekseenko Date: Wed, 27 Oct 2021 15:28:55 +0200 Subject: [PATCH] Set up event consumption counter in DomDecMpiTests Also: - Add a comment about the barrier introduced in 5d43901b (MR !2109). - Use DeviceStream::synchronize instead of marking an event and immediately waiting for it. Refs #3988, #4156. --- src/gromacs/domdec/tests/haloexchange_mpi.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/gromacs/domdec/tests/haloexchange_mpi.cpp b/src/gromacs/domdec/tests/haloexchange_mpi.cpp index fac6cc09ab..93aad8c288 100644 --- a/src/gromacs/domdec/tests/haloexchange_mpi.cpp +++ b/src/gromacs/domdec/tests/haloexchange_mpi.cpp @@ -53,6 +53,7 @@ #include "config.h" #include +#include #include #include @@ -140,7 +141,14 @@ void gpuHalo(gmx_domdec_t* dd, matrix box, HostVector* h_x, int numAtomsTo copyToDeviceBuffer(&d_x, h_x->data(), 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr); - GpuEventSynchronizer coordinatesReadyOnDeviceEvent; + const int numPulses = std::accumulate( + dd->comm->cd.begin(), dd->comm->cd.end(), 0, [](const int a, const auto& b) { + return a + b.numPulses(); + }); + const int numExtraConsumptions = GMX_THREAD_MPI ? 1 : 0; + // Will be consumed once for each pulse, and, with tMPI, once more for dim=0,pulse=0 case + GpuEventSynchronizer coordinatesReadyOnDeviceEvent(numPulses + numExtraConsumptions, + numPulses + numExtraConsumptions); coordinatesReadyOnDeviceEvent.markEvent(deviceStream); std::array, DIM> gpuHaloExchange; @@ -164,11 +172,10 @@ void gpuHalo(gmx_domdec_t* dd, matrix box, HostVector* h_x, int numAtomsTo gpuHaloExchange[d][pulse].communicateHaloCoordinates(box, &coordinatesReadyOnDeviceEvent); } } + // Barrier is needed to avoid other threads using events after its owner has exited and destroyed the context. MPI_Barrier(MPI_COMM_WORLD); - GpuEventSynchronizer haloCompletedEvent; - haloCompletedEvent.markEvent(deviceStream); - haloCompletedEvent.waitForEvent(); + deviceStream.synchronize(); // Copy results back to host copyFromDeviceBuffer( -- 2.22.0