#include "config.h"
#include <array>
+#include <numeric>
#include <vector>
#include <gtest/gtest.h>
copyToDeviceBuffer(&d_x, h_x->data(), 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr);
- GpuEventSynchronizer coordinatesReadyOnDeviceEvent;
+ const int numPulses = std::accumulate(
+ dd->comm->cd.begin(), dd->comm->cd.end(), 0, [](const int a, const auto& b) {
+ return a + b.numPulses();
+ });
+ const int numExtraConsumptions = GMX_THREAD_MPI ? 1 : 0;
+ // Will be consumed once for each pulse, and, with tMPI, once more for dim=0,pulse=0 case
+ GpuEventSynchronizer coordinatesReadyOnDeviceEvent(numPulses + numExtraConsumptions,
+ numPulses + numExtraConsumptions);
coordinatesReadyOnDeviceEvent.markEvent(deviceStream);
std::array<std::vector<GpuHaloExchange>, DIM> gpuHaloExchange;
{
for (int pulse = 0; pulse < dd->comm->cd[d].numPulses(); pulse++)
{
- gpuHaloExchange[d].push_back(GpuHaloExchange(
- dd, d, MPI_COMM_WORLD, deviceContext, deviceStream, deviceStream, pulse, nullptr));
+ gpuHaloExchange[d].push_back(
+ GpuHaloExchange(dd, d, MPI_COMM_WORLD, deviceContext, pulse, nullptr));
}
}
gpuHaloExchange[d][pulse].communicateHaloCoordinates(box, &coordinatesReadyOnDeviceEvent);
}
}
+ // Barrier is needed to avoid other threads using events after its owner has exited and destroyed the context.
+ MPI_Barrier(MPI_COMM_WORLD);
- GpuEventSynchronizer haloCompletedEvent;
- haloCompletedEvent.markEvent(deviceStream);
- haloCompletedEvent.waitForEvent();
+ deviceStream.synchronize();
// Copy results back to host
copyFromDeviceBuffer(
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- dd->neighbor[0][0] = (rank + 1) % 4;
- dd->neighbor[0][1] = (rank == 0) ? 3 : rank - 1;
+ const int numRanks = getNumberOfTestMpiRanks();
+ dd->neighbor[0][0] = (rank + 1) % numRanks;
+ dd->neighbor[0][1] = (rank == 0) ? (numRanks - 1) : rank - 1;
}
/*! \brief Define 2D rank topology with 4 MPI tasks
TEST(HaloExchangeTest, Coordinates1dHaloWith1Pulse)
{
- GMX_MPI_TEST(4);
+ GMX_MPI_TEST(RequireRankCount<4>);
// Set up atom data
const int numHomeAtoms = 10;
TEST(HaloExchangeTest, Coordinates1dHaloWith2Pulses)
{
- GMX_MPI_TEST(4);
+ GMX_MPI_TEST(RequireRankCount<4>);
// Set up atom data
const int numHomeAtoms = 10;
TEST(HaloExchangeTest, Coordinates2dHaloWith1PulseInEachDim)
{
- GMX_MPI_TEST(4);
+ GMX_MPI_TEST(RequireRankCount<4>);
// Set up atom data
const int numHomeAtoms = 10;
TEST(HaloExchangeTest, Coordinates2dHaloWith2PulsesInDim1)
{
- GMX_MPI_TEST(4);
+ GMX_MPI_TEST(RequireRankCount<4>);
// Set up atom data
const int numHomeAtoms = 10;