DeviceInformation is a basic contained and does not need the getters.
#include "config.h"
#include "gromacs/gpu_utils/device_context.h"
-#include "gromacs/gpu_utils/gputraits.h"
#include "gromacs/utility/classhelpers.h"
class DeviceContext;
return;
}
// Constructing contexts for all compatible GPUs - will be empty on non-GPU builds
- for (int gpuIndex : getCompatibleGpus(hardwareInfo_->gpu_info))
+ for (const DeviceInformation& compatibleDeviceInfo : getCompatibleDevices(hardwareInfo_->deviceInfoList))
{
- const DeviceInformation* deviceInfo = getDeviceInfo(hardwareInfo_->gpu_info, gpuIndex);
- init_gpu(deviceInfo);
-
- char stmp[200] = {};
- get_gpu_device_info_string(stmp, hardwareInfo_->gpu_info, gpuIndex);
- std::string description = "(GPU " + std::string(stmp) + ") ";
+ setActiveDevice(compatibleDeviceInfo);
+ std::string deviceDescription = getDeviceInformationString(compatibleDeviceInfo);
+ std::string description = "(GPU " + deviceDescription + ") ";
hardwareContexts_.emplace_back(std::make_unique<TestHardwareContext>(
- CodePath::GPU, description.c_str(), *deviceInfo));
+ CodePath::GPU, description.c_str(), compatibleDeviceInfo));
}
}
#include <gtest/gtest.h>
#include "gromacs/ewald/pme_gpu_program.h"
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/utility/gmxassert.h"
#include "testhardwarecontext.h"
device_stream_manager.cpp
hostallocator.cpp
gpu_utils.cpp
- gpu_testutils.cpp
)
if(GMX_GPU_OPENCL)
gmx_add_libgromacs_sources(
#include "device_stream.h"
-#include "gromacs/gpu_utils/gputraits.h"
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/gmxassert.h"
#include "gromacs/utility/stringutil.h"
#include "gromacs/gpu_utils/device_context.h"
#include "gromacs/gpu_utils/device_stream.h"
-#include "gromacs/gpu_utils/gputraits.h"
#include "gromacs/utility/enumerationhelpers.h"
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/gmxassert.h"
+++ /dev/null
-/*
- * This file is part of the GROMACS molecular simulation package.
- *
- * Copyright (c) 2019,2020, by the GROMACS development team, led by
- * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
- * and including many others, as listed in the AUTHORS file in the
- * top-level source directory and at http://www.gromacs.org.
- *
- * GROMACS is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2.1
- * of the License, or (at your option) any later version.
- *
- * GROMACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with GROMACS; if not, see
- * http://www.gnu.org/licenses, or write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * If you want to redistribute modifications to GROMACS, please
- * consider that scientific software is very special. Version
- * control is crucial - bugs must be traceable. We will be happy to
- * consider code for inclusion in the official distribution, but
- * derived work must not be called official GROMACS. Details are found
- * in the README & COPYING files - if they are missing, get the
- * official version at http://www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org.
- */
-/*! \internal \file
- * \brief Function definitions for GPU detection, specific for tests.
- *
- * \author Artem Zhmurov <zhmurov@gmail.com>
- */
-#include "gmxpre.h"
-
-#include "gpu_testutils.h"
-
-#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/gpu_hw_info.h"
-
-bool canComputeOnGpu()
-{
- bool canComputeOnGpu = false;
- gmx_gpu_info_t gpuInfo{};
- if (canPerformGpuDetection())
- {
- findGpus(&gpuInfo);
- canComputeOnGpu = !getCompatibleGpus(gpuInfo).empty();
- }
- free_gpu_info(&gpuInfo);
- return canComputeOnGpu;
-}
+++ /dev/null
-/*
- * This file is part of the GROMACS molecular simulation package.
- *
- * Copyright (c) 2019, by the GROMACS development team, led by
- * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
- * and including many others, as listed in the AUTHORS file in the
- * top-level source directory and at http://www.gromacs.org.
- *
- * GROMACS is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2.1
- * of the License, or (at your option) any later version.
- *
- * GROMACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with GROMACS; if not, see
- * http://www.gnu.org/licenses, or write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * If you want to redistribute modifications to GROMACS, please
- * consider that scientific software is very special. Version
- * control is crucial - bugs must be traceable. We will be happy to
- * consider code for inclusion in the official distribution, but
- * derived work must not be called official GROMACS. Details are found
- * in the README & COPYING files - if they are missing, get the
- * official version at http://www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org.
- */
-/*! \libinternal \file
- * \brief Declare functions for detection of GPU devices, specific for tests.
- *
- * \todo This should eventually go to src/testutils
- *
- * \author Artem Zhmurov <zhmurov@gmail.com>
- *
- * \inlibraryapi
- */
-
-#ifndef GMX_GPU_UTILS_GPU_TESTUTILS_H
-#define GMX_GPU_UTILS_GPU_TESTUTILS_H
-
-/*! \brief Checks if there is a compatible GPU to run the computations on
- *
- * There are several reasons why code can not rune on the GPU:
- * 1. The GPU can not be detected, because there is none in the system.
- * 2. GPU detection is disabled by GMX_DISABLE_GPU_DETECTION environmental variable.
- * 3. GPUs are detected, but none of them is compatible.
- * This function checks all these conditions and returns true only if there at least
- * one GPU that can be used for computations.
- *
- * \returns True, if there a GPU that can be used for computations
- */
-bool canComputeOnGpu();
-
-#endif // GMX_GPU_UTILS_GPU_TESTUTILS_H
#include "gpu_utils.h"
+#include "config.h"
+
#include <cassert>
-#include "gromacs/hardware/device_information.h"
#include "gromacs/utility/arrayref.h"
#include "gromacs/utility/smalloc.h"
#include "gromacs/utility/stringutil.h"
{
errorReasons.emplace_back("double precision");
}
- if (!c_binarySupportsGpus)
+ if (!GMX_GPU)
{
errorReasons.emplace_back("non-GPU build of GROMACS");
}
#include "gromacs/gpu_utils/device_context.h"
#include "gromacs/gpu_utils/device_stream.h"
#include "gromacs/gpu_utils/pmalloc_cuda.h"
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_information.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/utility/basedefinitions.h"
#include "gromacs/utility/cstringutil.h"
#include "gromacs/utility/exceptions.h"
*/
#include <cuda_runtime.h>
-#include "gromacs/hardware/gpu_hw_info.h"
-
//! Device texture for fast read-only data fetching
using DeviceTexture = cudaTextureObject_t;
*/
#include "gromacs/gpu_utils/gmxopencl.h"
-#include "gromacs/hardware/gpu_hw_info.h"
using DeviceTexture = void*;
#include "gromacs/gpu_utils/gputraits_ocl.h"
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/gmxassert.h"
+#include "gromacs/utility/stringutil.h"
enum class GpuApiCallBehavior;
// that we've called, so it is not very useful.
const bool useTiming = false;
- // TODO Is it enough to only test one device?
- for (const auto* deviceInfo : getDeviceInfos())
+ for (const auto& deviceInfo : getDeviceInfoList())
{
EXPECT_FALSE(deviceInfo == nullptr)
<< "Device information should be provided for the GPU builds.";
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2017,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
namespace gmx
{
-void doDeviceTransfers(const gmx_gpu_info_t& /*gpuInfo*/, ArrayRef<const char> input, ArrayRef<char> output)
+void doDeviceTransfers(const DeviceInformation& /* deviceInfo */,
+ ArrayRef<const char> input,
+ ArrayRef<char> output)
{
GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size");
// We can't have any valid GPUs for this build configuration.
#include "devicetransfers.h"
#include "gromacs/gpu_utils/cudautils.cuh"
-#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_information.h"
#include "gromacs/utility/arrayref.h"
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/gmxassert.h"
} // namespace
-void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef<const char> input, ArrayRef<char> output)
+void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef<const char> input, ArrayRef<char> output)
{
GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size");
- const auto compatibleGpus = getCompatibleGpus(gpuInfo);
- if (compatibleGpus.empty())
- {
- std::copy(input.begin(), input.end(), output.begin());
- return;
- }
cudaError_t status;
- const auto* device = getDeviceInfo(gpuInfo, compatibleGpus[0]);
- int oldDeviceId;
+ int oldDeviceId;
status = cudaGetDevice(&oldDeviceId);
throwUponFailure(status, "getting old device id");
- status = cudaSetDevice(device->id);
+ status = cudaSetDevice(deviceInfo.id);
throwUponFailure(status, "setting device id to the first compatible GPU");
void* devicePointer;
#ifndef GMX_GPU_UTILS_TESTS_DEVICETRANSFERS_H
#define GMX_GPU_UTILS_TESTS_DEVICETRANSFERS_H
-struct gmx_gpu_info_t;
+struct DeviceInformation;
namespace gmx
{
* do a simple host-side buffer copy instead.
*
* \throws InternalError Upon any GPU API error condition. */
-void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef<const char> input, ArrayRef<char> output);
+void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef<const char> input, ArrayRef<char> output);
} // namespace gmx
#include "gromacs/gpu_utils/gmxopencl.h"
#include "gromacs/gpu_utils/oclutils.h"
+#include "gromacs/hardware/device_information.h"
#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/gpu_hw_info.h"
#include "gromacs/utility/arrayref.h"
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/gmxassert.h"
} // namespace
-void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef<const char> input, ArrayRef<char> output)
+void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef<const char> input, ArrayRef<char> output)
{
GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size");
- const auto compatibleGpus = getCompatibleGpus(gpuInfo);
- if (compatibleGpus.empty())
- {
- std::copy(input.begin(), input.end(), output.begin());
- return;
- }
+
cl_int status;
- const auto* device = getDeviceInfo(gpuInfo, compatibleGpus[0]);
cl_context_properties properties[] = {
- CL_CONTEXT_PLATFORM, reinterpret_cast<cl_context_properties>(device->oclPlatformId), 0
+ CL_CONTEXT_PLATFORM, reinterpret_cast<cl_context_properties>(deviceInfo.oclPlatformId), 0
};
// Give uncrustify more space
- auto deviceId = device->oclDeviceId;
+ auto deviceId = deviceInfo.oclDeviceId;
auto context = clCreateContext(properties, 1, &deviceId, nullptr, nullptr, &status);
throwUponFailure(status, "creating context");
auto commandQueue = clCreateCommandQueue(context, deviceId, 0, &status);
#include <gtest/gtest.h>
+#include "gromacs/gpu_utils/gpu_utils.h"
+#include "gromacs/hardware/device_information.h"
#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/gpu_hw_info.h"
#include "gromacs/utility/smalloc.h"
namespace gmx
GpuTest::GpuTest()
{
- snew(gpuInfo_, 1);
- if (isGpuDetectionFunctional(nullptr))
+ if (canPerformDeviceDetection(nullptr))
{
- findGpus(gpuInfo_);
- compatibleGpuIds_ = getCompatibleGpus(*gpuInfo_);
+ deviceInfoList_ = findDevices();
}
// Failing to find valid GPUs does not require further action
}
-GpuTest::~GpuTest()
-{
- free_gpu_info(gpuInfo_);
- sfree(gpuInfo_);
-}
+GpuTest::~GpuTest() = default;
-bool GpuTest::haveCompatibleGpus() const
+std::vector<std::unique_ptr<DeviceInformation>>& GpuTest::getDeviceInfoList()
{
- return !compatibleGpuIds_.empty();
-}
-
-std::vector<const DeviceInformation*> GpuTest::getDeviceInfos() const
-{
- std::vector<const DeviceInformation*> deviceInfos;
- deviceInfos.reserve(compatibleGpuIds_.size());
- for (const auto& id : compatibleGpuIds_)
- {
- deviceInfos.emplace_back(getDeviceInfo(*gpuInfo_, id));
- }
- return deviceInfos;
+ return deviceInfoList_;
}
} // namespace test
#include <gtest/gtest.h>
+#include "gromacs/hardware/device_management.h"
+
struct DeviceInformation;
-struct gmx_gpu_info_t;
namespace gmx
{
class GpuTest : public ::testing::Test
{
public:
- //! Information about GPUs that are present.
- gmx_gpu_info_t* gpuInfo_;
- //! Contains the IDs of all compatible GPUs
- std::vector<int> compatibleGpuIds_;
+ //! List of all available devices
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoList_;
GpuTest();
~GpuTest() override;
- //! Return whether compatible GPUs were found
- bool haveCompatibleGpus() const;
//! Return a vector of handles, each to a device info for a compatible GPU.
- std::vector<const DeviceInformation*> getDeviceInfos() const;
+ std::vector<std::unique_ptr<DeviceInformation>>& getDeviceInfoList();
};
} // namespace test
//! Does a device transfer of \c input to the device in \c gpuInfo, and back to \c output.
template<typename T>
-void runTest(const gmx_gpu_info_t& gpuInfo, ArrayRef<T> input, ArrayRef<T> output)
+void runTest(const DeviceInformation& deviceInfo, ArrayRef<T> input, ArrayRef<T> output)
{
// Convert the views of input and output to flat non-const chars,
// so that there's no templating when we call doDeviceTransfers.
auto outputRef = charArrayRefFromArray(output.data(), output.size());
ASSERT_EQ(inputRef.size(), outputRef.size());
- doDeviceTransfers(gpuInfo, inputRef, outputRef);
+
+ doDeviceTransfers(deviceInfo, inputRef, outputRef);
compareViews(input, output);
}
TYPED_TEST(HostAllocatorTestCopyable, TransfersWithoutPinningWork)
{
- typename TestFixture::VectorType input;
- fillInput(&input, 1);
- typename TestFixture::VectorType output;
- output.resizeWithPadding(input.size());
+ for (const DeviceInformation& compatibleDeviceInfo : getCompatibleDevices(this->deviceInfoList_))
+ {
+ typename TestFixture::VectorType input;
+ fillInput(&input, 1);
+ typename TestFixture::VectorType output;
+ output.resizeWithPadding(input.size());
- runTest(*this->gpuInfo_, makeArrayRef(input), makeArrayRef(output));
+ runTest(compatibleDeviceInfo, makeArrayRef(input), makeArrayRef(output));
+ }
}
TYPED_TEST(HostAllocatorTestCopyable, FillInputAlsoWorksAfterCallingReserve)
TYPED_TEST(HostAllocatorTestCopyable, TransfersWithPinningWorkWithCuda)
{
- if (!this->haveCompatibleGpus())
+ for (auto& deviceInfo : this->deviceInfoList_)
{
- return;
+ typename TestFixture::VectorType input;
+ changePinningPolicy(&input, PinningPolicy::PinnedIfSupported);
+ fillInput(&input, 1);
+ typename TestFixture::VectorType output;
+ changePinningPolicy(&output, PinningPolicy::PinnedIfSupported);
+ output.resizeWithPadding(input.size());
+
+ runTest(*deviceInfo, makeArrayRef(input), makeArrayRef(output));
}
-
- typename TestFixture::VectorType input;
- changePinningPolicy(&input, PinningPolicy::PinnedIfSupported);
- fillInput(&input, 1);
- typename TestFixture::VectorType output;
- changePinningPolicy(&output, PinningPolicy::PinnedIfSupported);
- output.resizeWithPadding(input.size());
-
- runTest(*this->gpuInfo_, makeArrayRef(input), makeArrayRef(output));
}
//! Helper function for wrapping a call to isHostMemoryPinned.
TYPED_TEST(HostAllocatorTestCopyable, ManualPinningOperationsWorkWithCuda)
{
- if (!this->haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, DefaultContainerIsRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, NonpinnedContainerIsRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, PinningChangesAreRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
TEST_F(PinnedMemoryCheckerTest, PinnedCBufferIsRecognized)
{
- if (!haveCompatibleGpus())
+ if (!canComputeOnDevice())
{
return;
}
# include <gtest/gtest.h>
-# include "gromacs/gpu_utils/gpu_testutils.h"
+# include "gromacs/hardware/device_management.h"
# include "gromacs/utility/exceptions.h"
# include "testutils/testasserts.h"
TEST(GpuDataTypesCompatibilityTest, RVecAndFloat3OnDevice)
{
- if (canComputeOnGpu())
+ if (canComputeOnDevice())
{
std::vector<RVec> rVecOutput(rVecInput.size());
convertRVecToFloat3OnDevice(rVecOutput, rVecInput);
#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/fatalerror.h"
#include "gromacs/utility/gmxassert.h"
+#include "gromacs/utility/inmemoryserializer.h"
#include "gromacs/utility/logger.h"
#include "gromacs/utility/mutex.h"
#include "gromacs/utility/physicalnodecommunicator.h"
#include "architecture.h"
+#include "device_information.h"
#ifdef HAVE_UNISTD_H
# include <unistd.h> // sysconf()
{
}
-gmx_hw_info_t::~gmx_hw_info_t()
-{
- free_gpu_info(&gpu_info);
-}
+gmx_hw_info_t::~gmx_hw_info_t() = default;
namespace gmx
{
const PhysicalNodeCommunicator& physicalNodeComm,
compat::not_null<gmx_hw_info_t*> hardwareInfo)
{
- hardwareInfo->gpu_info.bDetectGPUs = canPerformGpuDetection();
-
- if (!hardwareInfo->gpu_info.bDetectGPUs)
+ std::string errorMessage;
+ if (!canPerformDeviceDetection(&errorMessage))
{
+ GMX_LOG(mdlog.info)
+ .asParagraph()
+ .appendTextFormatted(
+ "NOTE: Detection of GPUs failed. The API reported:\n"
+ " %s\n"
+ " GROMACS cannot run tasks on a GPU.",
+ errorMessage.c_str());
return;
}
if (isMasterRankOfPhysicalNode || allRanksMustDetectGpus)
{
std::string errorMessage;
- gpusCanBeDetected = isGpuDetectionFunctional(&errorMessage);
+ gpusCanBeDetected = isDeviceDetectionFunctional(&errorMessage);
if (!gpusCanBeDetected)
{
GMX_LOG(mdlog.info)
if (gpusCanBeDetected)
{
- findGpus(&hardwareInfo->gpu_info);
+ hardwareInfo->deviceInfoList = findDevices();
// No need to tell the user anything at this point, they get a
// hardware report later.
}
#if GMX_LIB_MPI
- if (!allRanksMustDetectGpus)
+ if (!allRanksMustDetectGpus && !hardwareInfo->deviceInfoList.empty())
{
- /* Broadcast the GPU info to the other ranks within this node */
- MPI_Bcast(&hardwareInfo->gpu_info.n_dev, 1, MPI_INT, 0, physicalNodeComm.comm_);
-
- if (hardwareInfo->gpu_info.n_dev > 0)
- {
- int dev_size;
+ gmx::InMemorySerializer writer;
+ serializeDeviceInformations(hardwareInfo->deviceInfoList, &writer);
+ auto buffer = writer.finishAndGetBuffer();
- dev_size = hardwareInfo->gpu_info.n_dev * sizeof_gpu_dev_info();
+ MPI_Bcast(buffer.data(), buffer.size(), MPI_BYTE, 0, physicalNodeComm.comm_);
- if (!isMasterRankOfPhysicalNode)
- {
- hardwareInfo->gpu_info.deviceInfo = (struct DeviceInformation*)malloc(dev_size);
- }
- MPI_Bcast(hardwareInfo->gpu_info.deviceInfo, dev_size, MPI_BYTE, 0, physicalNodeComm.comm_);
- MPI_Bcast(&hardwareInfo->gpu_info.n_dev_compatible, 1, MPI_INT, 0, physicalNodeComm.comm_);
- }
+ gmx::InMemoryDeserializer reader(buffer, false);
+ hardwareInfo->deviceInfoList = deserializeDeviceInformations(&writer);
}
#endif
}
&& (cpuInfo.model() == 1 || cpuInfo.model() == 17
|| cpuInfo.model() == 8 || cpuInfo.model() == 24))
|| cpuInfo.vendor() == CpuInfo::Vendor::Hygon);
+
+ int numCompatibleDevices = getCompatibleDevices(hardwareInfo->deviceInfoList).size();
#if GMX_LIB_MPI
- int nhwthread, ngpu, i;
+ int nhwthread;
int gpu_hash;
nhwthread = hardwareInfo->nthreads_hw_avail;
- ngpu = hardwareInfo->gpu_info.n_dev_compatible;
/* Create a unique hash of the GPU type(s) in this node */
gpu_hash = 0;
/* Here it might be better to only loop over the compatible GPU, but we
* don't have that information available and it would also require
* removing the device ID from the device info string.
*/
- for (i = 0; i < hardwareInfo->gpu_info.n_dev; i++)
+ for (const auto& deviceInfo : hardwareInfo->deviceInfoList)
{
- char stmp[STRLEN];
-
/* Since the device ID is incorporated in the hash, the order of
* the GPUs affects the hash. Also two identical GPUs won't give
* a gpu_hash of zero after XORing.
*/
- get_gpu_device_info_string(stmp, hardwareInfo->gpu_info, i);
- gpu_hash ^= gmx_string_fullhash_func(stmp, gmx_string_hash_init);
+ std::string deviceInfoString = getDeviceInformationString(*deviceInfo);
+ gpu_hash ^= gmx_string_fullhash_func(deviceInfoString.c_str(), gmx_string_hash_init);
}
constexpr int numElementsCounts = 4;
countsLocal[0] = 1;
countsLocal[1] = ncore;
countsLocal[2] = nhwthread;
- countsLocal[3] = ngpu;
+ countsLocal[3] = numCompatibleDevices;
}
MPI_Allreduce(countsLocal.data(), countsReduced.data(), countsLocal.size(), MPI_INT,
*/
maxMinLocal[0] = ncore;
maxMinLocal[1] = nhwthread;
- maxMinLocal[2] = ngpu;
+ maxMinLocal[2] = numCompatibleDevices;
maxMinLocal[3] = static_cast<int>(gmx::simdSuggested(cpuInfo));
maxMinLocal[4] = gpu_hash;
maxMinLocal[5] = -maxMinLocal[0];
hardwareInfo->nhwthread_tot = hardwareInfo->nthreads_hw_avail;
hardwareInfo->nhwthread_min = hardwareInfo->nthreads_hw_avail;
hardwareInfo->nhwthread_max = hardwareInfo->nthreads_hw_avail;
- hardwareInfo->ngpu_compatible_tot = hardwareInfo->gpu_info.n_dev_compatible;
- hardwareInfo->ngpu_compatible_min = hardwareInfo->gpu_info.n_dev_compatible;
- hardwareInfo->ngpu_compatible_max = hardwareInfo->gpu_info.n_dev_compatible;
+ hardwareInfo->ngpu_compatible_tot = numCompatibleDevices;
+ hardwareInfo->ngpu_compatible_min = numCompatibleDevices;
+ hardwareInfo->ngpu_compatible_max = numCompatibleDevices;
hardwareInfo->simd_suggest_min = static_cast<int>(simdSuggested(cpuInfo));
hardwareInfo->simd_suggest_max = static_cast<int>(simdSuggested(cpuInfo));
hardwareInfo->bIdenticalGPUs = TRUE;
hardwareInfo->nthreads_hw_avail = hardwareInfo->hardwareTopology->machine().logicalProcessorCount;
// Detect GPUs
- hardwareInfo->gpu_info.n_dev = 0;
- hardwareInfo->gpu_info.n_dev_compatible = 0;
- hardwareInfo->gpu_info.deviceInfo = nullptr;
-
gmx_detect_gpus(mdlog, physicalNodeComm, compat::make_not_null(hardwareInfo));
gmx_collect_hardware_mpi(*hardwareInfo->cpuInfo, physicalNodeComm, compat::make_not_null(hardwareInfo));
return g_hardwareInfo.get();
}
-bool compatibleGpusFound(const gmx_gpu_info_t& gpu_info)
-{
- return gpu_info.n_dev_compatible > 0;
-}
-
} // namespace gmx
#ifndef GMX_HARDWARE_DETECTHARDWARE_H
#define GMX_HARDWARE_DETECTHARDWARE_H
-struct gmx_gpu_info_t;
struct gmx_hw_info_t;
namespace gmx
gmx_hw_info_t* gmx_detect_hardware(const gmx::MDLogger& mdlog,
const PhysicalNodeCommunicator& physicalNodeComm);
-//! Return whether compatible GPUs were found.
-bool compatibleGpusFound(const gmx_gpu_info_t& gpu_info);
-
} // namespace gmx
#endif
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* the research papers on the package. Check out http://www.gromacs.org.
*/
/*! \libinternal \file
- * \brief Declares the GPU type traits for non-GPU builds.
+ * \brief Declares the GPU information structure and its helpers
*
+ * \author Anca Hamuraru <anca@streamcomputing.eu>
+ * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
+ * \author Teemu Virolainen <teemu@streamcomputing.eu>
* \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Szilárd Páll <pall.szilard@gmail.com>
* \author Artem Zhmurov <zhmurov@gmail.com>
- *
- * \inlibraryapi
- * \ingroup module_hardware
*/
#ifndef GMX_HARDWARE_DEVICE_INFORMATION_H
#define GMX_HARDWARE_DEVICE_INFORMATION_H
struct DeviceInformation
{
//! Device status.
- DeviceStatus stat;
+ DeviceStatus status;
//! ID of the device.
int id;
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team.
- * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
/*! \internal \file
* \brief Defines the CPU stubs for the device management.
*
+ * \author Anca Hamuraru <anca@streamcomputing.eu>
+ * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
+ * \author Teemu Virolainen <teemu@streamcomputing.eu>
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Szilárd Páll <pall.szilard@gmail.com>
* \author Artem Zhmurov <zhmurov@gmail.com>
*
* \ingroup module_hardware
#include "device_management.h"
-bool isGpuDetectionFunctional(std::string* errorMessage)
-{
- if (errorMessage != nullptr)
- {
- errorMessage->assign("GROMACS has been built without GPU support.");
- }
- return false;
-}
+#include "gromacs/gpu_utils/gputraits.h"
+#include "gromacs/utility/fatalerror.h"
-void findGpus(gmx_gpu_info_t* /* gpu_info */)
-{
- GMX_RELEASE_ASSERT(false, "Trying to initialize GPUs in the build that does not support them.");
-}
+#include "device_information.h"
-void init_gpu(const DeviceInformation* /* deviceInfo */)
+std::vector<std::unique_ptr<DeviceInformation>> findDevices()
{
- GMX_RELEASE_ASSERT(false, "Trying to initialize GPU in the build that does not support GPUs.");
+ return {};
}
-void free_gpu(const DeviceInformation* /* deviceInfo */) {}
+void setActiveDevice(const DeviceInformation& /* deviceInfo */) {}
-DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& /* gpu_info */, int /* deviceId */)
-{
- GMX_RELEASE_ASSERT(
- false, "Trying to get GPU device information in the build that does not support GPUs.");
- return nullptr;
-}
+void releaseDevice(DeviceInformation* /* deviceInfo */) {}
-void get_gpu_device_info_string(char* /* s */, const gmx_gpu_info_t& /* gpu_info */, int /* index */)
+std::string getDeviceInformationString(const DeviceInformation& /* deviceInfo */)
{
- GMX_RELEASE_ASSERT(
- false,
- "Trying to get the GPU device description in the build that does not support GPUs.");
+ gmx_fatal(FARGS, "Device information requested in CPU build.");
}
-size_t sizeof_gpu_dev_info()
+bool isDeviceDetectionFunctional(std::string* /* errorMessage */)
{
- return 0;
-}
-
-DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& /* gpu_info */, int /* index */)
-{
- return DeviceStatus::Nonexistent;
+ return false;
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team.
- * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
/*! \internal \file
* \brief Defines the CUDA implementations of the device management.
*
+ * \author Anca Hamuraru <anca@streamcomputing.eu>
+ * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
+ * \author Teemu Virolainen <teemu@streamcomputing.eu>
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Szilárd Páll <pall.szilard@gmail.com>
* \author Artem Zhmurov <zhmurov@gmail.com>
*
* \ingroup module_hardware
#include "gromacs/gpu_utils/cudautils.cuh"
#include "gromacs/gpu_utils/device_context.h"
#include "gromacs/gpu_utils/device_stream.h"
+#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/programcontext.h"
#include "gromacs/utility/smalloc.h"
+#include "gromacs/utility/stringutil.h"
+
+#include "device_information.h"
/*! \internal \brief
* Max number of devices supported by CUDA (for consistency checking).
*
* In reality it is 16 with CUDA <=v5.0, but let's stay on the safe side.
*/
-static int cuda_max_device_count = 32;
+static int c_cudaMaxDeviceCount = 32;
/** Dummy kernel used for sanity checking. */
-static __global__ void k_dummy_test(void) {}
+static __global__ void dummy_kernel(void) {}
static cudaError_t checkCompiledTargetCompatibility(int deviceId, const cudaDeviceProp& deviceProp)
{
cudaFuncAttributes attributes;
- cudaError_t stat = cudaFuncGetAttributes(&attributes, k_dummy_test);
+ cudaError_t stat = cudaFuncGetAttributes(&attributes, dummy_kernel);
if (cudaErrorInvalidDeviceFunction == stat)
{
}
/* things might go horribly wrong if cudart is not compatible with the driver */
- if (dev_count < 0 || dev_count > cuda_max_device_count)
+ if (dev_count < 0 || dev_count > c_cudaMaxDeviceCount)
{
return DeviceStatus::NonFunctional;
}
{
KernelLaunchConfig config;
config.blockSize[0] = 512;
- const auto dummyArguments = prepareGpuKernelArguments(k_dummy_test, config);
+ const auto dummyArguments = prepareGpuKernelArguments(dummy_kernel, config);
DeviceInformation deviceInfo;
const DeviceContext deviceContext(deviceInfo);
const DeviceStream deviceStream(deviceContext, DeviceStreamPriority::Normal, false);
- launchGpuKernel(k_dummy_test, config, deviceStream, nullptr, "Dummy kernel", dummyArguments);
+ launchGpuKernel(dummy_kernel, config, deviceStream, nullptr, "Dummy kernel", dummyArguments);
}
catch (gmx::GromacsException& ex)
{
return DeviceStatus::Compatible;
}
-/*! \brief Returns true if the gpu characterized by the device properties is
- * supported by the native gpu acceleration.
+/*! \brief Returns true if the gpu characterized by the device properties is supported
+ * by the native gpu acceleration.
*
- * \param[in] dev_prop the CUDA device properties of the gpus to test.
- * \returns true if the GPU properties passed indicate a compatible
- * GPU, otherwise false.
+ * \param[in] deviceProperties The CUDA device properties of the gpus to test.
+ * \returns True if the GPU properties passed indicate a compatible
+ * GPU, otherwise false.
*/
-static bool is_gmx_supported_gpu(const cudaDeviceProp& dev_prop)
+static bool isDeviceGenerationSupported(const cudaDeviceProp& deviceProperties)
{
- return (dev_prop.major >= 3);
+ return (deviceProperties.major >= 3);
}
/*! \brief Checks if a GPU with a given ID is supported by the native GROMACS acceleration.
*/
static DeviceStatus checkDeviceStatus(int deviceId, const cudaDeviceProp& deviceProp)
{
- if (!is_gmx_supported_gpu(deviceProp))
+ if (!isDeviceGenerationSupported(deviceProp))
{
return DeviceStatus::Incompatible;
}
return isDeviceFunctional(deviceId, deviceProp);
}
-bool isGpuDetectionFunctional(std::string* errorMessage)
+bool isDeviceDetectionFunctional(std::string* errorMessage)
{
cudaError_t stat;
int driverVersion = -1;
return true;
}
-void findGpus(gmx_gpu_info_t* gpu_info)
+std::vector<std::unique_ptr<DeviceInformation>> findDevices()
{
- assert(gpu_info);
-
- gpu_info->n_dev_compatible = 0;
-
- int ndev;
- cudaError_t stat = cudaGetDeviceCount(&ndev);
+ int numDevices;
+ cudaError_t stat = cudaGetDeviceCount(&numDevices);
if (stat != cudaSuccess)
{
GMX_THROW(gmx::InternalError(
- "Invalid call of findGpus() when CUDA API returned an error, perhaps "
- "canDetectGpus() was not called appropriately beforehand."));
+ "Invalid call of findDevices() when CUDA API returned an error, perhaps "
+ "canPerformDeviceDetection() was not called appropriately beforehand."));
}
// We expect to start device support/sanity checks with a clean runtime error state
gmx::ensureNoPendingCudaError("");
- DeviceInformation* devs;
- snew(devs, ndev);
- for (int i = 0; i < ndev; i++)
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoList(numDevices);
+ for (int i = 0; i < numDevices; i++)
{
cudaDeviceProp prop;
memset(&prop, 0, sizeof(cudaDeviceProp));
const DeviceStatus checkResult =
(stat != cudaSuccess) ? DeviceStatus::NonFunctional : checkDeviceStatus(i, prop);
- devs[i].id = i;
- devs[i].prop = prop;
- devs[i].stat = checkResult;
+ deviceInfoList[i] = std::make_unique<DeviceInformation>();
- if (checkResult == DeviceStatus::Compatible)
- {
- gpu_info->n_dev_compatible++;
- }
- else
+ deviceInfoList[i]->id = i;
+ deviceInfoList[i]->prop = prop;
+ deviceInfoList[i]->status = checkResult;
+
+ if (checkResult != DeviceStatus::Compatible)
{
// TODO:
// - we inspect the CUDA API state to retrieve and record any
if ((stat = cudaGetLastError()) != cudaSuccess)
{
gmx_warning("An error occurred while sanity checking device #%d; %s: %s",
- devs[i].id, cudaGetErrorName(stat), cudaGetErrorString(stat));
+ deviceInfoList[i]->id, cudaGetErrorName(stat), cudaGetErrorString(stat));
}
}
}
cudaGetErrorName(stat), cudaGetErrorString(stat))
.c_str());
- gpu_info->n_dev = ndev;
- gpu_info->deviceInfo = devs;
+ return deviceInfoList;
}
-void init_gpu(const DeviceInformation* deviceInfo)
+void setActiveDevice(const DeviceInformation& deviceInfo)
{
+ int deviceId = deviceInfo.id;
cudaError_t stat;
- assert(deviceInfo);
-
- stat = cudaSetDevice(deviceInfo->id);
+ stat = cudaSetDevice(deviceId);
if (stat != cudaSuccess)
{
- auto message = gmx::formatString("Failed to initialize GPU #%d", deviceInfo->id);
+ auto message = gmx::formatString("Failed to initialize GPU #%d", deviceId);
CU_RET_ERR(stat, message.c_str());
}
if (debug)
{
- fprintf(stderr, "Initialized GPU ID #%d: %s\n", deviceInfo->id, deviceInfo->prop.name);
+ fprintf(stderr, "Initialized GPU ID #%d: %s\n", deviceId, deviceInfo.prop.name);
}
}
-void free_gpu(const DeviceInformation* deviceInfo)
+void releaseDevice(DeviceInformation* deviceInfo)
{
- // One should only attempt to clear the device context when
- // it has been used, but currently the only way to know that a GPU
// device was used is that deviceInfo will be non-null.
- if (deviceInfo == nullptr)
+ if (deviceInfo != nullptr)
{
- return;
- }
-
- cudaError_t stat;
+ cudaError_t stat;
- if (debug)
- {
int gpuid;
stat = cudaGetDevice(&gpuid);
- CU_RET_ERR(stat, "cudaGetDevice failed");
- fprintf(stderr, "Cleaning up context on GPU ID #%d\n", gpuid);
- }
-
- stat = cudaDeviceReset();
- if (stat != cudaSuccess)
- {
- gmx_warning("Failed to free GPU #%d: %s", deviceInfo->id, cudaGetErrorString(stat));
- }
-}
+ if (stat == cudaSuccess)
+ {
+ if (debug)
+ {
+ fprintf(stderr, "Cleaning up context on GPU ID #%d\n", gpuid);
+ }
-DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId)
-{
- if (deviceId < 0 || deviceId >= gpu_info.n_dev)
- {
- gmx_incons("Invalid GPU deviceId requested");
+ stat = cudaDeviceReset();
+ if (stat != cudaSuccess)
+ {
+ gmx_warning("Failed to free GPU #%d: %s", gpuid, cudaGetErrorString(stat));
+ }
+ }
}
- return &gpu_info.deviceInfo[deviceId];
}
-void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index)
+std::string getDeviceInformationString(const DeviceInformation& deviceInfo)
{
- assert(s);
-
- if (index < 0 && index >= gpu_info.n_dev)
- {
- return;
- }
-
- DeviceInformation* dinfo = &gpu_info.deviceInfo[index];
-
- bool bGpuExists =
- (dinfo->stat != DeviceStatus::Nonexistent && dinfo->stat != DeviceStatus::NonFunctional);
+ bool gpuExists = (deviceInfo.status != DeviceStatus::Nonexistent
+ && deviceInfo.status != DeviceStatus::NonFunctional);
- if (!bGpuExists)
+ if (!gpuExists)
{
- sprintf(s, "#%d: %s, stat: %s", dinfo->id, "N/A", c_deviceStateString[dinfo->stat]);
+ return gmx::formatString("#%d: %s, stat: %s", deviceInfo.id, "N/A",
+ c_deviceStateString[deviceInfo.status]);
}
else
{
- sprintf(s, "#%d: NVIDIA %s, compute cap.: %d.%d, ECC: %3s, stat: %s", dinfo->id,
- dinfo->prop.name, dinfo->prop.major, dinfo->prop.minor,
- dinfo->prop.ECCEnabled ? "yes" : " no", c_deviceStateString[dinfo->stat]);
+ return gmx::formatString("#%d: NVIDIA %s, compute cap.: %d.%d, ECC: %3s, stat: %s",
+ deviceInfo.id, deviceInfo.prop.name, deviceInfo.prop.major,
+ deviceInfo.prop.minor, deviceInfo.prop.ECCEnabled ? "yes" : " no",
+ c_deviceStateString[deviceInfo.status]);
}
}
-
-size_t sizeof_gpu_dev_info(void)
-{
- return sizeof(DeviceInformation);
-}
-
-DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index)
-{
- return info.deviceInfo[index].stat;
-}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2020, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* the research papers on the package. Check out http://www.gromacs.org.
*/
/*! \libinternal \file
+ * \brief Declares functions to manage GPU resources.
*
- * \brief Implements the device management for OpenCL.
+ * This has several implementations: one for each supported GPU platform,
+ * and a stub implementation if the build does not support GPUs.
*
- * \author Artem Zhmurov <zhmurov@gmail.com>
+ * \author Anca Hamuraru <anca@streamcomputing.eu>
+ * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
+ * \author Teemu Virolainen <teemu@streamcomputing.eu>
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Szilárd Páll <pall.szilard@gmail.com>
+ * \author Artem Zhmurov <zhmurov@gmail.com>
*
* \inlibraryapi
* \ingroup module_hardware
#ifndef GMX_HARDWARE_DEVICE_MANAGEMENT_H
#define GMX_HARDWARE_DEVICE_MANAGEMENT_H
-#include "gmxpre.h"
-
+#include <memory>
#include <string>
#include <vector>
-#include "gromacs/hardware/device_information.h"
+#include "gromacs/utility/basedefinitions.h"
+#include "gromacs/utility/iserializer.h"
struct DeviceInformation;
-enum class DeviceStatus : int;
-struct gmx_gpu_info_t;
-/*! \brief Return whether GPUs can be detected
+/*! \brief Return whether GPUs can be detected.
+ *
+ * Returns true when this is a build of GROMACS configured to support
+ * GPU usage, GPU detection is not disabled by \c GMX_DISABLE_GPU_DETECTION
+ * environment variable and a valid device driver, ICD, and/or runtime was
+ * detected. Does not throw.
*
- * Returns true when this is a build of \Gromacs configured to support
- * GPU usage, GPU detection is not disabled by an environment variable
- * and a valid device driver, ICD, and/or runtime was detected.
- * Does not throw. */
-bool canPerformGpuDetection();
+ * \param[out] errorMessage When returning false on a build configured with
+ * GPU support and non-nullptr was passed,
+ * the string contains a descriptive message about
+ * why GPUs cannot be detected.
+ */
+bool canPerformDeviceDetection(std::string* errorMessage);
/*! \brief Return whether GPU detection is functioning correctly
*
- * Returns true when this is a build of \Gromacs configured to support
+ * Returns true when this is a build of GROMACS configured to support
* GPU usage, and a valid device driver, ICD, and/or runtime was detected.
*
* This function is not intended to be called from build
* the string contains a descriptive message about
* why GPUs cannot be detected.
*
- * Does not throw. */
-bool isGpuDetectionFunctional(std::string* errorMessage);
+ * Does not throw.
+ */
+bool isDeviceDetectionFunctional(std::string* errorMessage);
+
+/*! \brief Checks if one can compute on the GPU
+ *
+ * \returns True if the build supports GPUs and there are at least one available.
+ */
+bool canComputeOnDevice();
/*! \brief Find all GPUs in the system.
*
* Will detect every GPU supported by the device driver in use.
- * Must only be called if canPerformGpuDetection() has returned true.
- * This routine also checks for the compatibility of each and fill the
- * gpu_info->deviceInfo array with the required information on each the
- * device: ID, device properties, status.
+ * Must only be called if \c canPerformDeviceDetection() has returned true.
+ * This routine also checks for the compatibility of each device and fill the
+ * deviceInfo array with the required information on each device: ID, device
+ * properties, status.
*
* Note that this function leaves the GPU runtime API error state clean;
* this is implemented ATM in the CUDA flavor.
- * TODO: check if errors do propagate in OpenCL as they do in CUDA and
- * whether there is a mechanism to "clear" them.
*
- * \param[in] gpu_info pointer to structure holding GPU information.
+ * \todo: Check if errors do propagate in OpenCL as they do in CUDA and
+ * whether there is a mechanism to "clear" them.
+ *
+ * \return Standard vector with the list of devices found
*
- * \throws InternalError if a GPU API returns an unexpected failure (because
- * the call to canDetectGpus() should always prevent this occuring)
+ * \throws InternalError if a GPU API returns an unexpected failure (because
+ * the call to canDetectGpus() should always prevent this occuring)
*/
-void findGpus(gmx_gpu_info_t* gpu_info);
+std::vector<std::unique_ptr<DeviceInformation>> findDevices();
-/*! \brief Return a container of the detected GPUs that are compatible.
+/*! \brief Return a container of the detected GPU ids that are compatible.
*
* This function filters the result of the detection for compatible
* GPUs, based on the previously run compatibility tests.
*
- * \param[in] gpu_info Information detected about GPUs, including compatibility.
- * \return vector of IDs of GPUs already recorded as compatible */
-std::vector<int> getCompatibleGpus(const gmx_gpu_info_t& gpu_info);
-
-/*! \brief Return a string describing how compatible the GPU with given \c index is.
- *
- * \param[in] gpu_info Information about detected GPUs
- * \param[in] index index of GPU to ask about
- * \returns A null-terminated C string describing the compatibility status, useful for error messages.
- */
-const char* getGpuCompatibilityDescription(const gmx_gpu_info_t& gpu_info, int index);
-
-/*! \brief Frees the gpu_dev and dev_use array fields of \p gpu_info.
+ * \param[in] deviceInfoList An information on available devices.
*
- * \param[in] gpu_info pointer to structure holding GPU information
+ * \return Vector of DeviceInformations on GPUs recorded as compatible
*/
-void free_gpu_info(const gmx_gpu_info_t* gpu_info);
+std::vector<std::reference_wrapper<DeviceInformation>>
+getCompatibleDevices(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList);
-/*! \brief Initializes the GPU described by \c deviceInfo.
+/*! \brief Set the active GPU.
*
- * TODO Doxygen complains about these - probably a Doxygen bug, since
- * the patterns here are the same as elsewhere in this header.
+ * This sets the device for which the device information is passed active. Essential in CUDA, where
+ * the device buffers and kernel launches are not connected to the device context. In OpenCL, checks
+ * the device vendor and makes vendor-specific performance adjustments.
*
- * \param[in] deviceInfo device info of the GPU to initialize
+ * \param[in] deviceInfo Information on the device to be set.
*
* Issues a fatal error for any critical errors that occur during
* initialization.
*/
-void init_gpu(const DeviceInformation* deviceInfo);
+void setActiveDevice(const DeviceInformation& deviceInfo);
-/*! \brief Frees up the CUDA GPU used by the active context at the time of calling.
+/*! \brief Releases the GPU device used by the active context at the time of calling (CUDA only).
*
* If \c deviceInfo is nullptr, then it is understood that no device
* was selected so no context is active to be freed. Otherwise, the
* required anymore, because subsequent attempts to free memory
* associated with the context will otherwise fail.
*
- * Calls gmx_warning upon errors.
+ * Calls \c gmx_warning upon errors.
*
- * \param[in] deviceInfo device info of the GPU to clean up for
+ * \todo This should go through all the devices, not only the one currently active.
+ * Reseting only one device will not work, e.g. in CUDA tests.
*
- * \returns true if no error occurs during the freeing.
+ * \param[in] deviceInfo Information on the device to be released.
*/
-void free_gpu(const DeviceInformation* deviceInfo);
+void releaseDevice(DeviceInformation* deviceInfo);
-/*! \brief Return a pointer to the device info for \c deviceId
+/*! \brief Formats and returns a device information string for a given GPU.
+ *
+ * Given an index *directly* into the array of available GPUs, returns
+ * a formatted info string for the respective GPU which includes ID, name,
+ * compute capability, and detection status.
*
- * \param[in] gpu_info GPU info of all detected devices in the system.
- * \param[in] deviceId ID for the GPU device requested.
+ * \param[in] deviceInfo An information on device that is to be set.
*
- * \returns Pointer to the device info for \c deviceId.
+ * \returns A string describing the device.
*/
-DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId);
+std::string getDeviceInformationString(const DeviceInformation& deviceInfo);
-/*! \brief Formats and returns a device information string for a given GPU.
- *
- * Given an index *directly* into the array of available GPUs (gpu_dev)
- * returns a formatted info string for the respective GPU which includes
- * ID, name, compute capability, and detection status.
+/*! \brief Return a string describing how compatible the GPU with given \c deviceId is.
*
- * \param[out] s pointer to output string (has to be allocated externally)
- * \param[in] gpu_info Information about detected GPUs
- * \param[in] index an index *directly* into the array of available GPUs
+ * \param[in] deviceInfoList An information on available devices.
+ * \param[in] deviceId An index of the device to check
+ * \returns A string describing the compatibility status, useful for error messages.
*/
-void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index);
+std::string getDeviceCompatibilityDescription(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ int deviceId);
+/*! \brief Serialization of information on devices for MPI broadcasting.
+ *
+ * \param[in] deviceInfoList The vector with device informations to serialize.
+ * \param[in] serializer Serializing object.
+ */
+void serializeDeviceInformations(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ gmx::ISerializer* serializer);
-/*! \brief Returns the size of the gpu_dev_info struct.
+/*! \brief Deserialization of information on devices after MPI broadcasting.
*
- * The size of gpu_dev_info can be used for allocation and communication.
+ * \param[in] serializer Serializing object.
*
- * \returns size in bytes of gpu_dev_info
+ * \return deviceInfoList Deserialized vector with device informations.
*/
-size_t sizeof_gpu_dev_info();
-
-//! Get status of device with specified index
-DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index);
+std::vector<std::unique_ptr<DeviceInformation>> deserializeDeviceInformations(gmx::ISerializer* serializer);
#endif // GMX_HARDWARE_DEVICE_MANAGEMENT_H
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team.
- * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* the research papers on the package. Check out http://www.gromacs.org.
*/
/*! \internal \file
- * \brief Defines the implementations of the device management that are common for CPU, CUDA and OpenCL.
+ * \brief Defines the implementations of device management functions that
+ * are common for CPU, CUDA and OpenCL.
*
+ * \author Anca Hamuraru <anca@streamcomputing.eu>
+ * \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
+ * \author Teemu Virolainen <teemu@streamcomputing.eu>
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Szilárd Páll <pall.szilard@gmail.com>
* \author Artem Zhmurov <zhmurov@gmail.com>
*
* \ingroup module_hardware
*/
#include "gmxpre.h"
-#include <assert.h>
-
-#include "gromacs/hardware/device_information.h"
#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/gpu_hw_info.h"
-#include "gromacs/utility/smalloc.h"
+#include "gromacs/utility/fatalerror.h"
+
+#include "device_information.h"
-bool canPerformGpuDetection()
+bool canPerformDeviceDetection(std::string* errorMessage)
{
if (c_binarySupportsGpus && getenv("GMX_DISABLE_GPU_DETECTION") == nullptr)
{
- return isGpuDetectionFunctional(nullptr);
+ return isDeviceDetectionFunctional(errorMessage);
}
else
{
}
}
-std::vector<int> getCompatibleGpus(const gmx_gpu_info_t& gpu_info)
+bool canComputeOnDevice()
+{
+ bool canComputeOnDevice = false;
+ if (canPerformDeviceDetection(nullptr))
+ {
+ std::vector<std::unique_ptr<DeviceInformation>> devInfos = findDevices();
+ canComputeOnDevice = !getCompatibleDevices(devInfos).empty();
+ }
+ return canComputeOnDevice;
+}
+
+std::vector<std::reference_wrapper<DeviceInformation>>
+getCompatibleDevices(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList)
{
// Possible minor over-allocation here, but not important for anything
- std::vector<int> compatibleGpus;
- compatibleGpus.reserve(gpu_info.n_dev);
- for (int i = 0; i < gpu_info.n_dev; i++)
+ std::vector<std::reference_wrapper<DeviceInformation>> compatibleDeviceInfoList;
+ compatibleDeviceInfoList.reserve(deviceInfoList.size());
+ for (const auto& deviceInfo : deviceInfoList)
{
- assert(gpu_info.deviceInfo);
- if (gpu_info_get_stat(gpu_info, i) == DeviceStatus::Compatible)
+ if (deviceInfo->status == DeviceStatus::Compatible)
{
- compatibleGpus.push_back(i);
+ compatibleDeviceInfoList.emplace_back(*deviceInfo);
}
}
- return compatibleGpus;
+ return compatibleDeviceInfoList;
}
-const char* getGpuCompatibilityDescription(const gmx_gpu_info_t& gpu_info, int index)
+std::string getDeviceCompatibilityDescription(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ int deviceId)
{
- return (index >= gpu_info.n_dev ? c_deviceStateString[DeviceStatus::Nonexistent]
- : c_deviceStateString[gpu_info_get_stat(gpu_info, index)]);
+ return (deviceId >= static_cast<int>(deviceInfoList.size())
+ ? c_deviceStateString[DeviceStatus::Nonexistent]
+ : c_deviceStateString[deviceInfoList[deviceId]->status]);
}
-void free_gpu_info(const gmx_gpu_info_t* gpu_info)
+void serializeDeviceInformations(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ gmx::ISerializer* serializer)
{
- sfree(static_cast<void*>(gpu_info->deviceInfo)); // circumvent is_pod check in sfree
+ int numDevices = deviceInfoList.size();
+ serializer->doInt(&numDevices);
+ for (auto& deviceInfo : deviceInfoList)
+ {
+ serializer->doOpaque(reinterpret_cast<char*>(deviceInfo.get()), sizeof(DeviceInformation));
+ }
+}
+
+std::vector<std::unique_ptr<DeviceInformation>> deserializeDeviceInformations(gmx::ISerializer* serializer)
+{
+ int numDevices = 0;
+ serializer->doInt(&numDevices);
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoList(numDevices);
+ for (int i = 0; i < numDevices; i++)
+ {
+ deviceInfoList[i] = std::make_unique<DeviceInformation>();
+ serializer->doOpaque(reinterpret_cast<char*>(deviceInfoList[i].get()), sizeof(DeviceInformation));
+ }
+ return deviceInfoList;
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013,2014,2015,2016 by the GROMACS development team.
+ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team.
* Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* the research papers on the package. Check out http://www.gromacs.org.
*/
/*! \internal \file
- * \brief Define functions for detection and initialization for OpenCL devices.
+ * \brief Defines the OpenCL implementations of the device management.
*
* \author Anca Hamuraru <anca@streamcomputing.eu>
* \author Dimitrios Karkoulis <dimitris.karkoulis@gmail.com>
* \author Teemu Virolainen <teemu@streamcomputing.eu>
* \author Mark Abraham <mark.j.abraham@gmail.com>
* \author Szilárd Páll <pall.szilard@gmail.com>
+ * \author Artem Zhmurov <zhmurov@gmail.com>
+ *
+ * \ingroup module_hardware
*/
-
#include "gmxpre.h"
#include "config.h"
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <cstdio>
-#ifdef __APPLE__
-# include <sys/sysctl.h>
-#endif
-
-#include <memory.h>
-
-#include "gromacs/gpu_utils/ocl_compiler.h"
#include "gromacs/gpu_utils/oclraii.h"
#include "gromacs/gpu_utils/oclutils.h"
-#include "gromacs/hardware/device_information.h"
#include "gromacs/hardware/device_management.h"
-#include "gromacs/hardware/hw_info.h"
-#include "gromacs/utility/cstringutil.h"
-#include "gromacs/utility/exceptions.h"
#include "gromacs/utility/fatalerror.h"
#include "gromacs/utility/smalloc.h"
#include "gromacs/utility/stringutil.h"
+#include "device_information.h"
+
+namespace gmx
+{
+
+/*! \brief Returns an DeviceVendor value corresponding to the input OpenCL vendor name.
+ *
+ * \returns DeviceVendor value for the input vendor name
+ */
+static DeviceVendor getDeviceVendor(const char* vendorName)
+{
+ if (vendorName)
+ {
+ if (strstr(vendorName, "NVIDIA"))
+ {
+ return DeviceVendor::Nvidia;
+ }
+ else if (strstr(vendorName, "AMD") || strstr(vendorName, "Advanced Micro Devices"))
+ {
+ return DeviceVendor::Amd;
+ }
+ else if (strstr(vendorName, "Intel"))
+ {
+ return DeviceVendor::Intel;
+ }
+ }
+ return DeviceVendor::Unknown;
+}
+
/*! \brief Return true if executing on compatible OS for AMD OpenCL.
*
* This is assumed to be true for OS X version of at least 10.10.4 and
* all other OS flavors.
*
- * Uses the BSD sysctl() interfaces to extract the kernel version.
- *
* \return true if version is 14.4 or later (= OS X version 10.10.4),
* or OS is not Darwin.
*/
size_t len = sizeof(kernelVersion);
mib[0] = CTL_KERN;
- mib[1] = KERN_OSRELEASE;
-
- sysctl(mib, sizeof(mib) / sizeof(mib[0]), kernelVersion, &len, NULL, 0);
int major = strtod(kernelVersion, NULL);
int minor = strtod(strchr(kernelVersion, '.') + 1, NULL);
#endif
}
-namespace gmx
+/*!
+ * \brief Checks that device \c deviceInfo is compatible with GROMACS.
+ *
+ * Vendor and OpenCL version support checks are executed an the result
+ * of these returned.
+ *
+ * \param[in] deviceInfo The device info pointer.
+ * \returns The result of the compatibility checks.
+ */
+static DeviceStatus isDeviceSupported(const DeviceInformation& deviceInfo)
{
+ if (getenv("GMX_OCL_DISABLE_COMPATIBILITY_CHECK") != nullptr)
+ {
+ // Assume the device is compatible because checking has been disabled.
+ return DeviceStatus::Compatible;
+ }
+
+ // OpenCL device version check, ensure >= REQUIRED_OPENCL_MIN_VERSION
+ constexpr unsigned int minVersionMajor = REQUIRED_OPENCL_MIN_VERSION_MAJOR;
+ constexpr unsigned int minVersionMinor = REQUIRED_OPENCL_MIN_VERSION_MINOR;
+
+ // Based on the OpenCL spec we're checking the version supported by
+ // the device which has the following format:
+ // OpenCL<space><major_version.minor_version><space><vendor-specific information>
+ unsigned int deviceVersionMinor, deviceVersionMajor;
+ const int valuesScanned = std::sscanf(deviceInfo.device_version, "OpenCL %u.%u",
+ &deviceVersionMajor, &deviceVersionMinor);
+ const bool versionLargeEnough =
+ ((valuesScanned == 2)
+ && ((deviceVersionMajor > minVersionMajor)
+ || (deviceVersionMajor == minVersionMajor && deviceVersionMinor >= minVersionMinor)));
+ if (!versionLargeEnough)
+ {
+ return DeviceStatus::Incompatible;
+ }
+
+ /* Only AMD, Intel, and NVIDIA GPUs are supported for now */
+ switch (deviceInfo.deviceVendor)
+ {
+ case DeviceVendor::Nvidia: return DeviceStatus::Compatible;
+ case DeviceVendor::Amd:
+ return runningOnCompatibleOSForAmd() ? DeviceStatus::Compatible : DeviceStatus::Incompatible;
+ case DeviceVendor::Intel:
+ return GMX_OPENCL_NB_CLUSTER_SIZE == 4 ? DeviceStatus::Compatible
+ : DeviceStatus::IncompatibleClusterSize;
+ default: return DeviceStatus::Incompatible;
+ }
+}
/*! \brief Make an error string following an OpenCL API call.
*
* It is meant to be called with \p status != CL_SUCCESS, but it will
* work correctly even if it is called with no OpenCL failure.
*
+ * \todo Make use of this function more.
+ *
* \param[in] message Supplies context, e.g. the name of the API call that returned the error.
* \param[in] status OpenCL API status code
* \returns A string describing the OpenCL error.
*/
-static std::string makeOpenClInternalErrorString(const char* message, cl_int status)
+inline std::string makeOpenClInternalErrorString(const char* message, cl_int status)
{
if (message != nullptr)
{
- return formatString("%s did %ssucceed %d: %s", message, ((status != CL_SUCCESS) ? "not " : ""),
- status, ocl_get_error_string(status).c_str());
+ return gmx::formatString("%s did %ssucceed %d: %s", message,
+ ((status != CL_SUCCESS) ? "not " : ""), status,
+ ocl_get_error_string(status).c_str());
}
else
{
- return formatString("%sOpenCL error encountered %d: %s", ((status != CL_SUCCESS) ? "" : "No "),
- status, ocl_get_error_string(status).c_str());
+ return gmx::formatString("%sOpenCL error encountered %d: %s",
+ ((status != CL_SUCCESS) ? "" : "No "), status,
+ ocl_get_error_string(status).c_str());
}
}
* \throws std::bad_alloc When out of memory.
* \returns Whether the device passed sanity checks
*/
-static bool isDeviceFunctional(const DeviceInformation* deviceInfo, std::string* errorMessage)
+static bool isDeviceFunctional(const DeviceInformation& deviceInfo, std::string* errorMessage)
{
cl_context_properties properties[] = {
- CL_CONTEXT_PLATFORM, reinterpret_cast<cl_context_properties>(deviceInfo->oclPlatformId), 0
+ CL_CONTEXT_PLATFORM, reinterpret_cast<cl_context_properties>(deviceInfo.oclPlatformId), 0
};
// uncrustify spacing
cl_int status;
- auto deviceId = deviceInfo->oclDeviceId;
+ auto deviceId = deviceInfo.oclDeviceId;
ClContext context(clCreateContext(properties, 1, &deviceId, nullptr, nullptr, &status));
if (status != CL_SUCCESS)
{
return true;
}
-/*!
- * \brief Checks that device \c deviceInfo is compatible with GROMACS.
- *
- * Vendor and OpenCL version support checks are executed an the result
- * of these returned.
- *
- * \param[in] deviceInfo The device info pointer.
- * \returns The result of the compatibility checks.
- */
-static DeviceStatus isDeviceSupported(const DeviceInformation* deviceInfo)
-{
- if (getenv("GMX_OCL_DISABLE_COMPATIBILITY_CHECK") != nullptr)
- {
- // Assume the device is compatible because checking has been disabled.
- return DeviceStatus::Compatible;
- }
-
- // OpenCL device version check, ensure >= REQUIRED_OPENCL_MIN_VERSION
- constexpr unsigned int minVersionMajor = REQUIRED_OPENCL_MIN_VERSION_MAJOR;
- constexpr unsigned int minVersionMinor = REQUIRED_OPENCL_MIN_VERSION_MINOR;
-
- // Based on the OpenCL spec we're checking the version supported by
- // the device which has the following format:
- // OpenCL<space><major_version.minor_version><space><vendor-specific information>
- unsigned int deviceVersionMinor, deviceVersionMajor;
- const int valuesScanned = std::sscanf(deviceInfo->device_version, "OpenCL %u.%u",
- &deviceVersionMajor, &deviceVersionMinor);
- const bool versionLargeEnough =
- ((valuesScanned == 2)
- && ((deviceVersionMajor > minVersionMajor)
- || (deviceVersionMajor == minVersionMajor && deviceVersionMinor >= minVersionMinor)));
- if (!versionLargeEnough)
- {
- return DeviceStatus::Incompatible;
- }
-
- /* Only AMD, Intel, and NVIDIA GPUs are supported for now */
- switch (deviceInfo->deviceVendor)
- {
- case DeviceVendor::Nvidia: return DeviceStatus::Compatible;
- case DeviceVendor::Amd:
- return runningOnCompatibleOSForAmd() ? DeviceStatus::Compatible : DeviceStatus::Incompatible;
- case DeviceVendor::Intel:
- return GMX_OPENCL_NB_CLUSTER_SIZE == 4 ? DeviceStatus::Compatible
- : DeviceStatus::IncompatibleClusterSize;
- default: return DeviceStatus::Incompatible;
- }
-}
-
-
/*! \brief Check whether the \c ocl_gpu_device is suitable for use by mdrun
*
* Runs sanity checks: checking that the runtime can compile a dummy kernel
* \returns A DeviceStatus to indicate if the GPU device is supported and if it was able to run
* basic functionality checks.
*/
-static DeviceStatus checkGpu(size_t deviceId, const DeviceInformation* deviceInfo)
+static DeviceStatus checkGpu(size_t deviceId, const DeviceInformation& deviceInfo)
{
DeviceStatus supportStatus = isDeviceSupported(deviceInfo);
} // namespace gmx
-/*! \brief Returns an DeviceVendor value corresponding to the input OpenCL vendor name.
- *
- * \param[in] vendorName String with OpenCL vendor name.
- * \returns DeviceVendor value for the input vendor name
- */
-static DeviceVendor getDeviceVendor(const char* vendorName)
-{
- if (vendorName)
- {
- if (strstr(vendorName, "NVIDIA"))
- {
- return DeviceVendor::Nvidia;
- }
- else if (strstr(vendorName, "AMD") || strstr(vendorName, "Advanced Micro Devices"))
- {
- return DeviceVendor::Amd;
- }
- else if (strstr(vendorName, "Intel"))
- {
- return DeviceVendor::Intel;
- }
- }
- return DeviceVendor::Unknown;
-}
-
-bool isGpuDetectionFunctional(std::string* errorMessage)
+bool isDeviceDetectionFunctional(std::string* errorMessage)
{
cl_uint numPlatforms;
cl_int status = clGetPlatformIDs(0, nullptr, &numPlatforms);
return foundPlatform;
}
-void findGpus(gmx_gpu_info_t* gpu_info)
+std::vector<std::unique_ptr<DeviceInformation>> findDevices()
{
cl_uint ocl_platform_count;
cl_platform_id* ocl_platform_ids;
req_dev_type = CL_DEVICE_TYPE_CPU;
}
+ int numDevices = 0;
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoList(0);
+
while (true)
{
cl_int status = clGetPlatformIDs(0, nullptr, &ocl_platform_count);
if (1 <= ocl_device_count)
{
- gpu_info->n_dev += ocl_device_count;
+ numDevices += ocl_device_count;
}
}
- if (1 > gpu_info->n_dev)
+ if (1 > numDevices)
{
break;
}
- snew(gpu_info->deviceInfo, gpu_info->n_dev);
+ deviceInfoList.resize(numDevices);
{
int device_index;
cl_device_id* ocl_device_ids;
- snew(ocl_device_ids, gpu_info->n_dev);
+ snew(ocl_device_ids, numDevices);
device_index = 0;
for (unsigned int i = 0; i < ocl_platform_count; i++)
/* If requesting req_dev_type devices fails, just go to the next platform */
if (CL_SUCCESS
- != clGetDeviceIDs(ocl_platform_ids[i], req_dev_type, gpu_info->n_dev,
- ocl_device_ids, &ocl_device_count))
+ != clGetDeviceIDs(ocl_platform_ids[i], req_dev_type, numDevices, ocl_device_ids,
+ &ocl_device_count))
{
continue;
}
for (unsigned int j = 0; j < ocl_device_count; j++)
{
- gpu_info->deviceInfo[device_index].oclPlatformId = ocl_platform_ids[i];
- gpu_info->deviceInfo[device_index].oclDeviceId = ocl_device_ids[j];
+ deviceInfoList[device_index] = std::make_unique<DeviceInformation>();
- gpu_info->deviceInfo[device_index].device_name[0] = 0;
+ deviceInfoList[device_index]->id = device_index;
+
+ deviceInfoList[device_index]->oclPlatformId = ocl_platform_ids[i];
+ deviceInfoList[device_index]->oclDeviceId = ocl_device_ids[j];
+
+ deviceInfoList[device_index]->device_name[0] = 0;
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_NAME,
- sizeof(gpu_info->deviceInfo[device_index].device_name),
- gpu_info->deviceInfo[device_index].device_name, nullptr);
+ sizeof(deviceInfoList[device_index]->device_name),
+ deviceInfoList[device_index]->device_name, nullptr);
- gpu_info->deviceInfo[device_index].device_version[0] = 0;
+ deviceInfoList[device_index]->device_version[0] = 0;
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_VERSION,
- sizeof(gpu_info->deviceInfo[device_index].device_version),
- gpu_info->deviceInfo[device_index].device_version, nullptr);
+ sizeof(deviceInfoList[device_index]->device_version),
+ deviceInfoList[device_index]->device_version, nullptr);
- gpu_info->deviceInfo[device_index].vendorName[0] = 0;
+ deviceInfoList[device_index]->vendorName[0] = 0;
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_VENDOR,
- sizeof(gpu_info->deviceInfo[device_index].vendorName),
- gpu_info->deviceInfo[device_index].vendorName, nullptr);
+ sizeof(deviceInfoList[device_index]->vendorName),
+ deviceInfoList[device_index]->vendorName, nullptr);
- gpu_info->deviceInfo[device_index].compute_units = 0;
+ deviceInfoList[device_index]->compute_units = 0;
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_COMPUTE_UNITS,
- sizeof(gpu_info->deviceInfo[device_index].compute_units),
- &(gpu_info->deviceInfo[device_index].compute_units), nullptr);
+ sizeof(deviceInfoList[device_index]->compute_units),
+ &(deviceInfoList[device_index]->compute_units), nullptr);
- gpu_info->deviceInfo[device_index].adress_bits = 0;
+ deviceInfoList[device_index]->adress_bits = 0;
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_ADDRESS_BITS,
- sizeof(gpu_info->deviceInfo[device_index].adress_bits),
- &(gpu_info->deviceInfo[device_index].adress_bits), nullptr);
+ sizeof(deviceInfoList[device_index]->adress_bits),
+ &(deviceInfoList[device_index]->adress_bits), nullptr);
- gpu_info->deviceInfo[device_index].deviceVendor =
- getDeviceVendor(gpu_info->deviceInfo[device_index].vendorName);
+ deviceInfoList[device_index]->deviceVendor =
+ gmx::getDeviceVendor(deviceInfoList[device_index]->vendorName);
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_WORK_ITEM_SIZES, 3 * sizeof(size_t),
- &gpu_info->deviceInfo[device_index].maxWorkItemSizes, nullptr);
+ &deviceInfoList[device_index]->maxWorkItemSizes, nullptr);
clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t),
- &gpu_info->deviceInfo[device_index].maxWorkGroupSize, nullptr);
-
- gpu_info->deviceInfo[device_index].stat =
- gmx::checkGpu(device_index, gpu_info->deviceInfo + device_index);
+ &deviceInfoList[device_index]->maxWorkGroupSize, nullptr);
- if (DeviceStatus::Compatible == gpu_info->deviceInfo[device_index].stat)
- {
- gpu_info->n_dev_compatible++;
- }
+ deviceInfoList[device_index]->status =
+ gmx::checkGpu(device_index, *deviceInfoList[device_index]);
device_index++;
}
}
- gpu_info->n_dev = device_index;
+ numDevices = device_index;
/* Dummy sort of devices - AMD first, then NVIDIA, then Intel */
// TODO: Sort devices based on performance.
- if (0 < gpu_info->n_dev)
+ if (0 < numDevices)
{
int last = -1;
- for (int i = 0; i < gpu_info->n_dev; i++)
+ for (int i = 0; i < numDevices; i++)
{
- if (gpu_info->deviceInfo[i].deviceVendor == DeviceVendor::Amd)
+ if (deviceInfoList[i]->deviceVendor == DeviceVendor::Amd)
{
last++;
if (last < i)
{
- std::swap(gpu_info->deviceInfo[i], gpu_info->deviceInfo[last]);
+ std::swap(deviceInfoList[i], deviceInfoList[last]);
}
}
}
/* if more than 1 device left to be sorted */
- if ((gpu_info->n_dev - 1 - last) > 1)
+ if ((numDevices - 1 - last) > 1)
{
- for (int i = 0; i < gpu_info->n_dev; i++)
+ for (int i = 0; i < numDevices; i++)
{
- if (gpu_info->deviceInfo[i].deviceVendor == DeviceVendor::Nvidia)
+ if (deviceInfoList[i]->deviceVendor == DeviceVendor::Nvidia)
{
last++;
if (last < i)
{
- std::swap(gpu_info->deviceInfo[i], gpu_info->deviceInfo[last]);
+ std::swap(deviceInfoList[i], deviceInfoList[last]);
}
}
}
}
sfree(ocl_platform_ids);
+ return deviceInfoList;
}
-void init_gpu(const DeviceInformation* deviceInfo)
+void setActiveDevice(const DeviceInformation& deviceInfo)
{
- assert(deviceInfo);
-
// If the device is NVIDIA, for safety reasons we disable the JIT
// caching as this is known to be broken at least until driver 364.19;
// the cache does not always get regenerated when the source code changes,
// e.g. if the path to the kernel sources remains the same
- if (deviceInfo->deviceVendor == DeviceVendor::Nvidia)
+ if (deviceInfo.deviceVendor == DeviceVendor::Nvidia)
{
// Ignore return values, failing to set the variable does not mean
// that something will go wrong later.
}
}
-void free_gpu(const DeviceInformation* /* deviceInfo */) {}
-
-DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId)
-{
- if (deviceId < 0 || deviceId >= gpu_info.n_dev)
- {
- gmx_incons("Invalid GPU deviceId requested");
- }
- return &gpu_info.deviceInfo[deviceId];
-}
+void releaseDevice(DeviceInformation* /* deviceInfo */) {}
-void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index)
+std::string getDeviceInformationString(const DeviceInformation& deviceInfo)
{
- assert(s);
+ bool gpuExists = (deviceInfo.status != DeviceStatus::Nonexistent
+ && deviceInfo.status != DeviceStatus::NonFunctional);
- if (index < 0 && index >= gpu_info.n_dev)
+ if (!gpuExists)
{
- return;
- }
-
- DeviceInformation* dinfo = &gpu_info.deviceInfo[index];
-
- bool bGpuExists =
- (dinfo->stat != DeviceStatus::Nonexistent && dinfo->stat != DeviceStatus::NonFunctional);
-
- if (!bGpuExists)
- {
- sprintf(s, "#%d: %s, stat: %s", index, "N/A", c_deviceStateString[dinfo->stat]);
+ return gmx::formatString("#%d: %s, status: %s", deviceInfo.id, "N/A",
+ c_deviceStateString[deviceInfo.status]);
}
else
{
- sprintf(s, "#%d: name: %s, vendor: %s, device version: %s, stat: %s", index, dinfo->device_name,
- dinfo->vendorName, dinfo->device_version, c_deviceStateString[dinfo->stat]);
+ return gmx::formatString("#%d: name: %s, vendor: %s, device version: %s, status: %s",
+ deviceInfo.id, deviceInfo.device_name, deviceInfo.vendorName,
+ deviceInfo.device_version, c_deviceStateString[deviceInfo.status]);
}
}
-
-size_t sizeof_gpu_dev_info()
-{
- return sizeof(DeviceInformation);
-}
-
-DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index)
-{
- return info.deviceInfo[index].stat;
-}
+++ /dev/null
-/*
- * This file is part of the GROMACS molecular simulation package.
- *
- * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team.
- * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by
- * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
- * and including many others, as listed in the AUTHORS file in the
- * top-level source directory and at http://www.gromacs.org.
- *
- * GROMACS is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2.1
- * of the License, or (at your option) any later version.
- *
- * GROMACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with GROMACS; if not, see
- * http://www.gnu.org/licenses, or write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * If you want to redistribute modifications to GROMACS, please
- * consider that scientific software is very special. Version
- * control is crucial - bugs must be traceable. We will be happy to
- * consider code for inclusion in the official distribution, but
- * derived work must not be called official GROMACS. Details are found
- * in the README & COPYING files - if they are missing, get the
- * official version at http://www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org.
- */
-#ifndef GMX_HARDWARE_GPU_HW_INFO_H
-#define GMX_HARDWARE_GPU_HW_INFO_H
-
-#include "gromacs/utility/basedefinitions.h"
-#include "gromacs/utility/enumerationhelpers.h"
-
-struct DeviceInformation;
-
-/*! \brief Information about GPU devices on this physical node.
- *
- * Includes either CUDA or OpenCL devices. The gmx_hardware_detect
- * module initializes it.
- *
- * \todo Use a std::vector */
-struct gmx_gpu_info_t
-{
- //! Did we attempt GPU detection?
- gmx_bool bDetectGPUs;
- //! Total number of GPU devices detected on this physical node
- int n_dev;
- //! Information about each GPU device detected on this physical node
- DeviceInformation* deviceInfo;
- //! Number of GPU devices detected on this physical node that are compatible.
- int n_dev_compatible;
-};
-
-#endif
#include <string>
#include <vector>
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/utility/basedefinitions.h"
namespace gmx
class CpuInfo;
class HardwareTopology;
} // namespace gmx
+struct DeviceInformation;
/* Hardware information structure with CPU and GPU information.
* It is initialized by gmx_detect_hardware().
~gmx_hw_info_t();
/* Data for our local physical node */
- //! Information about GPUs detected on this physical node
- gmx_gpu_info_t gpu_info;
/*! \brief Number of hardware threads available.
*
std::unique_ptr<gmx::CpuInfo> cpuInfo; /* Information about CPU capabilities */
std::unique_ptr<gmx::HardwareTopology> hardwareTopology; /* Information about hardware topology */
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoList; /* Information about GPUs detected on this physical node */
/* Data reduced through MPI over all physical nodes */
/*! \internal \brief
* Returns the GPU information text, one GPU per line.
*/
-static std::string sprint_gpus(const gmx_gpu_info_t& gpu_info)
+static std::string sprint_gpus(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList)
{
- char stmp[STRLEN];
- std::vector<std::string> gpuStrings;
- for (int i = 0; i < gpu_info.n_dev; i++)
+ std::vector<std::string> gpuStrings(0);
+ for (const auto& deviceInfo : deviceInfoList)
{
- get_gpu_device_info_string(stmp, gpu_info, i);
- gpuStrings.push_back(gmx::formatString(" %s", stmp));
+ gpuStrings.emplace_back(" " + getDeviceInformationString(*deviceInfo));
}
return gmx::joinStrings(gpuStrings, "\n");
}
s += gmx::formatString(" %d cores,", hwinfo->ncore_tot);
}
s += gmx::formatString(" %d logical cores", hwinfo->nhwthread_tot);
- if (hwinfo->gpu_info.bDetectGPUs)
+ if (canPerformDeviceDetection(nullptr))
{
s += gmx::formatString(", %d compatible GPU%s", hwinfo->ngpu_compatible_tot,
hwinfo->ngpu_compatible_tot == 1 ? "" : "s");
}
}
- if (bGPUBinary && hwinfo->gpu_info.n_dev > 0)
+ if (bGPUBinary && !hwinfo->deviceInfoList.empty())
{
s += gmx::formatString(" GPU info:\n");
- s += gmx::formatString(" Number of GPUs detected: %d\n", hwinfo->gpu_info.n_dev);
- s += sprint_gpus(hwinfo->gpu_info) + "\n";
+ s += gmx::formatString(" Number of GPUs detected: %d\n",
+ static_cast<int>(hwinfo->deviceInfoList.size()));
+ s += sprint_gpus(hwinfo->deviceInfoList) + "\n";
}
return s;
}
gmx_add_unit_test(HardwareUnitTests hardware-test
CPP_SOURCE_FILES
cpuinfo.cpp
+ device_management.cpp
hardwaretopology.cpp
)
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2020, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*! \internal \file
+ * \brief
+ * Tests for DevicesManager
+ *
+ * \author Artem Zhmurov <zhmurov@gmail.com>
+ * \ingroup module_hardware
+ */
+#include "gmxpre.h"
+
+#include "gromacs/hardware/device_management.h"
+
+#include "config.h"
+
+#include <algorithm>
+
+#include <gtest/gtest.h>
+
+#include "gromacs/hardware/device_information.h"
+#include "gromacs/utility/inmemoryserializer.h"
+#include "gromacs/utility/stringutil.h"
+
+namespace
+{
+
+TEST(DevicesManagerTest, Serialization)
+{
+ if (canPerformDeviceDetection(nullptr))
+ {
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoListIn = findDevices();
+ gmx::InMemorySerializer writer;
+ serializeDeviceInformations(deviceInfoListIn, &writer);
+ auto buffer = writer.finishAndGetBuffer();
+
+ gmx::InMemoryDeserializer reader(buffer, false);
+ std::vector<std::unique_ptr<DeviceInformation>> deviceInfoListOut =
+ deserializeDeviceInformations(&reader);
+
+ EXPECT_EQ(deviceInfoListOut.size(), deviceInfoListIn.size())
+ << "Number of accessible devices changed after serialization/deserialization.";
+
+ for (int deviceId = 0; deviceId < static_cast<int>(deviceInfoListIn.size()); deviceId++)
+ {
+ EXPECT_FALSE(deviceInfoListIn[deviceId] == nullptr) << gmx::formatString(
+ "Device #%d information is nullptr before serialization.", deviceId);
+ EXPECT_FALSE(deviceInfoListOut[deviceId] == nullptr) << gmx::formatString(
+ "Device #%d information is nullptr after serialization.", deviceId);
+
+ const DeviceInformation& deviceInfoIn = *deviceInfoListIn[deviceId];
+ const DeviceInformation& deviceInfoOut = *deviceInfoListOut[deviceId];
+ EXPECT_EQ(deviceInfoIn.status, deviceInfoOut.status) << gmx::formatString(
+ "Device status changed after serialization/deserialization for device #%d.", deviceId);
+
+ EXPECT_EQ(deviceInfoIn.id, deviceInfoOut.id) << gmx::formatString(
+ "Device id changed after serialization/deserialization for device #%d.", deviceId);
+
+#if GMX_GPU_OPENCL
+ EXPECT_EQ(deviceInfoIn.oclPlatformId, deviceInfoOut.oclPlatformId) << gmx::formatString(
+ "Device OpenCL platform ID changed after serialization/deserialization for "
+ "device "
+ "#%d.",
+ deviceId);
+
+#endif // GMX_GPU_OPENCL
+ }
+ }
+}
+
+} // namespace
struct t_forcerec;
struct t_filenm;
struct t_inputrec;
-struct gmx_gpu_info_t;
struct gmx_localtop_t;
struct gmx_mtop_t;
struct gmx_wallcycle;
{
runnersNames.emplace_back("SHAKE");
runnersNames.emplace_back("LINCS");
- if (GMX_GPU_CUDA && canComputeOnGpu())
+ if (GMX_GPU_CUDA && canComputeOnDevice())
{
runnersNames.emplace_back("LINCS_GPU");
}
#include <vector>
#include "gromacs/gmxlib/nrnb.h"
-#include "gromacs/gpu_utils/gpu_testutils.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/math/paddedvector.h"
#include "gromacs/math/vec.h"
#include "gromacs/math/vectypes.h"
#include <vector>
#include "gromacs/gpu_utils/devicebuffer.cuh"
-#include "gromacs/hardware/device_management.h"
+#include "gromacs/hardware/device_information.h"
#include "gromacs/mdlib/lincs_gpu.cuh"
#include "gromacs/pbcutil/pbc.h"
#include "gromacs/utility/unique_cptr.h"
#include <gtest/gtest.h>
-#include "gromacs/gpu_utils/gpu_testutils.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/math/vec.h"
#include "gromacs/math/vectypes.h"
#include "gromacs/mdtypes/mdatom.h"
// All runners should be registered here under appropriate conditions
//
s_runners_["LeapFrogSimple"] = integrateLeapFrogSimple;
- if (GMX_GPU_CUDA && canComputeOnGpu())
+ if (GMX_GPU_CUDA && canComputeOnDevice())
{
s_runners_["LeapFrogGpu"] = integrateLeapFrogGpu;
}
#include <gtest/gtest.h>
-#include "gromacs/gpu_utils/gpu_testutils.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/math/vec.h"
#include "gromacs/math/vectypes.h"
#include "gromacs/mdtypes/mdatom.h"
//! Store whether any compatible GPUs exist.
static bool s_hasCompatibleGpus;
//! Before any test is run, work out whether any compatible GPUs exist.
- static void SetUpTestCase() { s_hasCompatibleGpus = canComputeOnGpu(); }
+ static void SetUpTestCase() { s_hasCompatibleGpus = canComputeOnDevice(); }
};
bool SettleTest::s_hasCompatibleGpus = false;
#include <vector>
#include "gromacs/gpu_utils/devicebuffer.cuh"
-#include "gromacs/hardware/device_management.h"
+#include "gromacs/hardware/device_information.h"
#include "gromacs/mdlib/settle_gpu.cuh"
#include "gromacs/utility/unique_cptr.h"
// there is a CUDA-capable device available.
GMX_RELEASE_ASSERT(GMX_GPU_CUDA, "CUDA version of SETTLE was called from non-CUDA build.");
- // TODO: Here we should check that at least 1 suitable GPU is available
- GMX_RELEASE_ASSERT(canPerformGpuDetection(), "Can't detect CUDA-capable GPUs.");
-
DeviceInformation deviceInfo;
const DeviceContext deviceContext(deviceInfo);
const DeviceStream deviceStream(deviceContext, DeviceStreamPriority::Normal, false);
gmx_print_detected_hardware(fplog, isSimulationMasterRank && isMasterSim(ms), mdlog, hwinfo);
- std::vector<int> gpuIdsToUse = makeGpuIdsToUse(hwinfo->gpu_info, hw_opt.gpuIdsAvailable);
+ std::vector<int> gpuIdsToUse = makeGpuIdsToUse(hwinfo->deviceInfoList, hw_opt.gpuIdsAvailable);
// Print citation requests after all software/hardware printing
pleaseCiteGromacs(fplog);
}
// FIXME: this is only here to manually unpin mdAtoms->chargeA_ and state->x,
- // before we destroy the GPU context(s) in free_gpu().
+ // before we destroy the GPU context(s)
// Pinned buffers are associated with contexts in CUDA.
// As soon as we destroy GPU contexts after mdrunner() exits, these lines should go.
mdAtoms.reset(nullptr);
sfree(disresdata);
sfree(oriresdata);
- if (hwinfo->gpu_info.n_dev > 0)
+ if (!hwinfo->deviceInfoList.empty())
{
/* stop the GPU profiler (only CUDA) */
stopGpuProfiler();
}
/* With tMPI we need to wait for all ranks to finish deallocation before
- * destroying the CUDA context in free_gpu() as some tMPI ranks may be sharing
+ * destroying the CUDA context as some tMPI ranks may be sharing
* GPU and context.
*
- * This is not a concern in OpenCL where we use one context per rank which
- * is freed in nbnxn_gpu_free().
+ * This is not a concern in OpenCL where we use one context per rank.
*
* Note: it is safe to not call the barrier on the ranks which do not use GPU,
* but it is easier and more futureproof to call it on the whole node.
{
physicalNodeComm.barrier();
}
-
- free_gpu(deviceInfo);
+ releaseDevice(deviceInfo);
/* Does what it says */
print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime());
# include "gromacs/gpu_utils/device_stream_manager.h"
# include "gromacs/gpu_utils/devicebuffer.h"
-# include "gromacs/gpu_utils/gputraits.h"
# include "gromacs/math/vectypes.h"
# include "gromacs/mdtypes/state_propagator_data_gpu.h"
# include "gromacs/timing/wallcycle.h"
// TODO Remove this comment when the above order issue is resolved
#include "gromacs/gpu_utils/cudautils.cuh"
+#include "gromacs/gpu_utils/device_context.h"
#include "gromacs/gpu_utils/device_stream_manager.h"
#include "gromacs/gpu_utils/gpu_utils.h"
#include "gromacs/gpu_utils/gpueventsynchronizer.cuh"
#include "gromacs/gpu_utils/pmalloc_cuda.h"
#include "gromacs/hardware/device_information.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/math/vectypes.h"
#include "gromacs/mdlib/force_flags.h"
#include "gromacs/mdtypes/interaction_const.h"
#include "gromacs/mdtypes/locality.h"
struct NbnxmGpu;
-struct gmx_gpu_info_t;
struct DeviceInformation;
struct gmx_wallclock_gpu_nbnxn_t;
struct nbnxn_atomdata_t;
#include "gromacs/gpu_utils/device_stream_manager.h"
#include "gromacs/gpu_utils/oclutils.h"
#include "gromacs/hardware/device_information.h"
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/math/vectypes.h"
#include "gromacs/mdlib/force_flags.h"
#include "gromacs/mdtypes/interaction_const.h"
userGpuTaskAssignment.size(), host, numGpuTasksOnThisNode)));
}
// Did the user choose compatible GPUs?
- checkUserGpuIds(hardwareInfo.gpu_info, gpuIdsToUse, userGpuTaskAssignment);
+ checkUserGpuIds(hardwareInfo.deviceInfoList, gpuIdsToUse, userGpuTaskAssignment);
gpuIdsForTaskAssignment = userGpuTaskAssignment;
}
if (gpuTaskMapping != gpuTaskAssignment.end())
{
*deviceId = gpuTaskMapping->deviceId_;
- deviceInfo = getDeviceInfo(hardwareInfo_.gpu_info, *deviceId);
- init_gpu(deviceInfo);
+ deviceInfo = hardwareInfo_.deviceInfoList[*deviceId].get();
+ setActiveDevice(*deviceInfo);
}
return deviceInfo;
}
#include <string>
#include <vector>
+#include "gromacs/hardware/device_information.h"
#include "gromacs/hardware/device_management.h"
#include "gromacs/hardware/hw_info.h"
#include "gromacs/utility/exceptions.h"
return digits;
}
-std::vector<int> makeGpuIdsToUse(const gmx_gpu_info_t& gpuInfo, const std::string& gpuIdsAvailableString)
+std::vector<int> makeGpuIdsToUse(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ const std::string& gpuIdsAvailableString)
{
- auto compatibleGpus = getCompatibleGpus(gpuInfo);
- std::vector<int> gpuIdsAvailable = parseUserGpuIdString(gpuIdsAvailableString);
+ auto compatibleDeviceInfoList = getCompatibleDevices(deviceInfoList);
+ std::vector<int> gpuIdsAvailable = parseUserGpuIdString(gpuIdsAvailableString);
+ std::vector<int> gpuIdsToUse;
if (gpuIdsAvailable.empty())
{
- return compatibleGpus;
+ for (const auto& compatibleDeviceInfo : compatibleDeviceInfoList)
+ {
+ gpuIdsToUse.emplace_back(compatibleDeviceInfo.get().id);
+ }
+ return gpuIdsToUse;
}
- std::vector<int> gpuIdsToUse;
gpuIdsToUse.reserve(gpuIdsAvailable.size());
std::vector<int> availableGpuIdsThatAreIncompatible;
for (const auto& availableGpuId : gpuIdsAvailable)
{
bool availableGpuIsCompatible = false;
- for (const auto& compatibleGpuId : compatibleGpus)
+ for (const auto& compatibleDeviceInfo : compatibleDeviceInfoList)
{
- if (availableGpuId == compatibleGpuId)
+ if (availableGpuId == compatibleDeviceInfo.get().id)
{
availableGpuIsCompatible = true;
break;
return formatAndJoin(resultGpuIds, ",", StringFormatter("%d"));
}
-void checkUserGpuIds(const gmx_gpu_info_t& gpu_info,
- const std::vector<int>& compatibleGpus,
- const std::vector<int>& gpuIds)
+void checkUserGpuIds(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ const std::vector<int>& compatibleGpus,
+ const std::vector<int>& gpuIds)
{
bool foundIncompatibleGpuIds = false;
std::string message =
{
foundIncompatibleGpuIds = true;
message += gmx::formatString(" GPU #%d: %s\n", gpuId,
- getGpuCompatibilityDescription(gpu_info, gpuId));
+ getDeviceCompatibilityDescription(deviceInfoList, gpuId).c_str());
}
}
if (foundIncompatibleGpuIds)
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include <cstddef>
+#include <memory>
#include <string>
#include <vector>
#include "gromacs/utility/arrayref.h"
-struct gmx_gpu_info_t;
+struct DeviceInformation;
namespace gmx
{
* all compatible GPUs on this physical node. Otherwise, check the
* user specified compatible GPUs and return their IDs.
*
- * \param[in] gpuInfo Information detected about GPUs on this physical node
+ * \param[in] deviceInfoList Information on the GPUs on this physical node.
* \param[in] gpuIdsAvailableString String like "013" or "0,1,3" typically
* supplied by the user to mdrun -gpu_id.
* Must contain only unique decimal digits, or only decimal
* InvalidInputError If gpuIdsAvailableString specifies GPU IDs that are
* not compatible.
*/
-std::vector<int> makeGpuIdsToUse(const gmx_gpu_info_t& gpuInfo, const std::string& gpuIdsAvailableString);
+std::vector<int> makeGpuIdsToUse(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ const std::string& gpuIdsAvailableString);
/*! \brief Parse a GPU ID specifier string into a container describing device ID to task mapping.
*
* infrastructure to do a good job of coordinating error messages and
* behaviour across MPMD ranks and multiple simulations.
*
- * \param[in] gpu_info Information detected about GPUs
+ * \param[in] deviceInfoList Information on the GPUs on this physical node.
* \param[in] compatibleGpus Vector of GPUs that are compatible
* \param[in] gpuIds The GPU IDs selected by the user.
*
* \throws std::bad_alloc If out of memory
* InconsistentInputError If the assigned GPUs are not valid
*/
-void checkUserGpuIds(const gmx_gpu_info_t& gpu_info,
- const std::vector<int>& compatibleGpus,
- const std::vector<int>& gpuIds);
+void checkUserGpuIds(const std::vector<std::unique_ptr<DeviceInformation>>& deviceInfoList,
+ const std::vector<int>& compatibleGpus,
+ const std::vector<int>& gpuIds);
} // namespace gmx
#include <gtest/gtest-spi.h>
#include "gromacs/ewald/pme.h"
-#include "gromacs/gpu_utils/gpu_testutils.h"
#include "gromacs/hardware/detecthardware.h"
-#include "gromacs/hardware/gpu_hw_info.h"
+#include "gromacs/hardware/device_management.h"
#include "gromacs/trajectory/energyframe.h"
#include "gromacs/utility/cstringutil.h"
#include "gromacs/utility/gmxmpi.h"
void PmeTest::SetUpTestCase()
{
- s_hasCompatibleGpus = canComputeOnGpu();
+ s_hasCompatibleGpus = canComputeOnDevice();
}
void PmeTest::runTest(const RunModesList& runModes)
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff