/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2020, by the GROMACS development team, led by
+ * Copyright (c) 2020,2021, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "config.h"
#include <array>
+#include <vector>
#include <gtest/gtest.h>
#if GMX_GPU_CUDA
# include "gromacs/gpu_utils/device_stream.h"
# include "gromacs/gpu_utils/devicebuffer.h"
-# include "gromacs/gpu_utils/gpueventsynchronizer.cuh"
#endif
+#include "gromacs/gpu_utils/gpueventsynchronizer.h"
#include "gromacs/gpu_utils/hostallocator.h"
#include "gromacs/mdtypes/inputrec.h"
GpuEventSynchronizer coordinatesReadyOnDeviceEvent;
coordinatesReadyOnDeviceEvent.markEvent(deviceStream);
+ std::array<std::vector<GpuHaloExchange>, DIM> gpuHaloExchange;
+
+ // Create halo exchange objects
+ for (int d = 0; d < dd->ndim; d++)
+ {
+ for (int pulse = 0; pulse < dd->comm->cd[d].numPulses(); pulse++)
+ {
+ gpuHaloExchange[d].push_back(
+ GpuHaloExchange(dd, d, MPI_COMM_WORLD, deviceContext, pulse, nullptr));
+ }
+ }
+
// Perform GPU halo exchange
for (int d = 0; d < dd->ndim; d++)
{
for (int pulse = 0; pulse < dd->comm->cd[d].numPulses(); pulse++)
{
- GpuHaloExchange gpuHaloExchange(dd, d, MPI_COMM_WORLD, deviceContext, deviceStream,
- deviceStream, pulse, nullptr);
- gpuHaloExchange.reinitHalo(d_x, nullptr);
- gpuHaloExchange.communicateHaloCoordinates(box, &coordinatesReadyOnDeviceEvent);
+ gpuHaloExchange[d][pulse].reinitHalo(d_x, nullptr);
+ gpuHaloExchange[d][pulse].communicateHaloCoordinates(box, &coordinatesReadyOnDeviceEvent);
}
}
+ MPI_Barrier(MPI_COMM_WORLD);
GpuEventSynchronizer haloCompletedEvent;
haloCompletedEvent.markEvent(deviceStream);
haloCompletedEvent.waitForEvent();
// Copy results back to host
- copyFromDeviceBuffer(h_x->data(), &d_x, 0, numAtomsTotal, deviceStream,
- GpuApiCallBehavior::Sync, nullptr);
+ copyFromDeviceBuffer(
+ h_x->data(), &d_x, 0, numAtomsTotal, deviceStream, GpuApiCallBehavior::Sync, nullptr);
freeDeviceBuffer(d_x);
#else