From: Artem Zhmurov Date: Mon, 7 Sep 2020 06:09:04 +0000 (+0000) Subject: Access the device status directly, remove the getter X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=9a9745f278dd8e88feef83f027c56551687ec42c;p=alexxy%2Fgromacs.git Access the device status directly, remove the getter DeviceInformation is a basic contained and does not need the getters. --- diff --git a/src/gromacs/ewald/pme_gpu_program_impl.h b/src/gromacs/ewald/pme_gpu_program_impl.h index 254a1ab215..6255e46054 100644 --- a/src/gromacs/ewald/pme_gpu_program_impl.h +++ b/src/gromacs/ewald/pme_gpu_program_impl.h @@ -45,7 +45,6 @@ #include "config.h" #include "gromacs/gpu_utils/device_context.h" -#include "gromacs/gpu_utils/gputraits.h" #include "gromacs/utility/classhelpers.h" class DeviceContext; diff --git a/src/gromacs/ewald/tests/testhardwarecontexts.cpp b/src/gromacs/ewald/tests/testhardwarecontexts.cpp index 6ae36951d5..5b7cb05327 100644 --- a/src/gromacs/ewald/tests/testhardwarecontexts.cpp +++ b/src/gromacs/ewald/tests/testhardwarecontexts.cpp @@ -105,16 +105,13 @@ void PmeTestEnvironment::SetUp() return; } // Constructing contexts for all compatible GPUs - will be empty on non-GPU builds - for (int gpuIndex : getCompatibleGpus(hardwareInfo_->gpu_info)) + for (const DeviceInformation& compatibleDeviceInfo : getCompatibleDevices(hardwareInfo_->deviceInfoList)) { - const DeviceInformation* deviceInfo = getDeviceInfo(hardwareInfo_->gpu_info, gpuIndex); - init_gpu(deviceInfo); - - char stmp[200] = {}; - get_gpu_device_info_string(stmp, hardwareInfo_->gpu_info, gpuIndex); - std::string description = "(GPU " + std::string(stmp) + ") "; + setActiveDevice(compatibleDeviceInfo); + std::string deviceDescription = getDeviceInformationString(compatibleDeviceInfo); + std::string description = "(GPU " + deviceDescription + ") "; hardwareContexts_.emplace_back(std::make_unique( - CodePath::GPU, description.c_str(), *deviceInfo)); + CodePath::GPU, description.c_str(), compatibleDeviceInfo)); } } diff --git a/src/gromacs/ewald/tests/testhardwarecontexts.h b/src/gromacs/ewald/tests/testhardwarecontexts.h index 6a1450fe79..42d7245a8e 100644 --- a/src/gromacs/ewald/tests/testhardwarecontexts.h +++ b/src/gromacs/ewald/tests/testhardwarecontexts.h @@ -49,7 +49,7 @@ #include #include "gromacs/ewald/pme_gpu_program.h" -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/utility/gmxassert.h" #include "testhardwarecontext.h" diff --git a/src/gromacs/gpu_utils/CMakeLists.txt b/src/gromacs/gpu_utils/CMakeLists.txt index a85efdeda1..9fea648387 100644 --- a/src/gromacs/gpu_utils/CMakeLists.txt +++ b/src/gromacs/gpu_utils/CMakeLists.txt @@ -41,7 +41,6 @@ gmx_add_libgromacs_sources( device_stream_manager.cpp hostallocator.cpp gpu_utils.cpp - gpu_testutils.cpp ) if(GMX_GPU_OPENCL) gmx_add_libgromacs_sources( diff --git a/src/gromacs/gpu_utils/device_stream.cu b/src/gromacs/gpu_utils/device_stream.cu index acb2bbdc9e..5cdc5bb20b 100644 --- a/src/gromacs/gpu_utils/device_stream.cu +++ b/src/gromacs/gpu_utils/device_stream.cu @@ -44,7 +44,6 @@ #include "device_stream.h" -#include "gromacs/gpu_utils/gputraits.h" #include "gromacs/utility/exceptions.h" #include "gromacs/utility/gmxassert.h" #include "gromacs/utility/stringutil.h" diff --git a/src/gromacs/gpu_utils/device_stream_manager.cpp b/src/gromacs/gpu_utils/device_stream_manager.cpp index 8c7457a3d3..96d3119b7c 100644 --- a/src/gromacs/gpu_utils/device_stream_manager.cpp +++ b/src/gromacs/gpu_utils/device_stream_manager.cpp @@ -47,7 +47,6 @@ #include "gromacs/gpu_utils/device_context.h" #include "gromacs/gpu_utils/device_stream.h" -#include "gromacs/gpu_utils/gputraits.h" #include "gromacs/utility/enumerationhelpers.h" #include "gromacs/utility/exceptions.h" #include "gromacs/utility/gmxassert.h" diff --git a/src/gromacs/gpu_utils/gpu_testutils.cpp b/src/gromacs/gpu_utils/gpu_testutils.cpp deleted file mode 100644 index 99b173c4a6..0000000000 --- a/src/gromacs/gpu_utils/gpu_testutils.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * This file is part of the GROMACS molecular simulation package. - * - * Copyright (c) 2019,2020, by the GROMACS development team, led by - * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, - * and including many others, as listed in the AUTHORS file in the - * top-level source directory and at http://www.gromacs.org. - * - * GROMACS is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * GROMACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with GROMACS; if not, see - * http://www.gnu.org/licenses, or write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * If you want to redistribute modifications to GROMACS, please - * consider that scientific software is very special. Version - * control is crucial - bugs must be traceable. We will be happy to - * consider code for inclusion in the official distribution, but - * derived work must not be called official GROMACS. Details are found - * in the README & COPYING files - if they are missing, get the - * official version at http://www.gromacs.org. - * - * To help us fund GROMACS development, we humbly ask that you cite - * the research papers on the package. Check out http://www.gromacs.org. - */ -/*! \internal \file - * \brief Function definitions for GPU detection, specific for tests. - * - * \author Artem Zhmurov - */ -#include "gmxpre.h" - -#include "gpu_testutils.h" - -#include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/gpu_hw_info.h" - -bool canComputeOnGpu() -{ - bool canComputeOnGpu = false; - gmx_gpu_info_t gpuInfo{}; - if (canPerformGpuDetection()) - { - findGpus(&gpuInfo); - canComputeOnGpu = !getCompatibleGpus(gpuInfo).empty(); - } - free_gpu_info(&gpuInfo); - return canComputeOnGpu; -} diff --git a/src/gromacs/gpu_utils/gpu_testutils.h b/src/gromacs/gpu_utils/gpu_testutils.h deleted file mode 100644 index 1ea82278e4..0000000000 --- a/src/gromacs/gpu_utils/gpu_testutils.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of the GROMACS molecular simulation package. - * - * Copyright (c) 2019, by the GROMACS development team, led by - * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, - * and including many others, as listed in the AUTHORS file in the - * top-level source directory and at http://www.gromacs.org. - * - * GROMACS is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * GROMACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with GROMACS; if not, see - * http://www.gnu.org/licenses, or write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * If you want to redistribute modifications to GROMACS, please - * consider that scientific software is very special. Version - * control is crucial - bugs must be traceable. We will be happy to - * consider code for inclusion in the official distribution, but - * derived work must not be called official GROMACS. Details are found - * in the README & COPYING files - if they are missing, get the - * official version at http://www.gromacs.org. - * - * To help us fund GROMACS development, we humbly ask that you cite - * the research papers on the package. Check out http://www.gromacs.org. - */ -/*! \libinternal \file - * \brief Declare functions for detection of GPU devices, specific for tests. - * - * \todo This should eventually go to src/testutils - * - * \author Artem Zhmurov - * - * \inlibraryapi - */ - -#ifndef GMX_GPU_UTILS_GPU_TESTUTILS_H -#define GMX_GPU_UTILS_GPU_TESTUTILS_H - -/*! \brief Checks if there is a compatible GPU to run the computations on - * - * There are several reasons why code can not rune on the GPU: - * 1. The GPU can not be detected, because there is none in the system. - * 2. GPU detection is disabled by GMX_DISABLE_GPU_DETECTION environmental variable. - * 3. GPUs are detected, but none of them is compatible. - * This function checks all these conditions and returns true only if there at least - * one GPU that can be used for computations. - * - * \returns True, if there a GPU that can be used for computations - */ -bool canComputeOnGpu(); - -#endif // GMX_GPU_UTILS_GPU_TESTUTILS_H diff --git a/src/gromacs/gpu_utils/gpu_utils.cpp b/src/gromacs/gpu_utils/gpu_utils.cpp index a8eb03b23a..1379ba83ce 100644 --- a/src/gromacs/gpu_utils/gpu_utils.cpp +++ b/src/gromacs/gpu_utils/gpu_utils.cpp @@ -41,9 +41,10 @@ #include "gpu_utils.h" +#include "config.h" + #include -#include "gromacs/hardware/device_information.h" #include "gromacs/utility/arrayref.h" #include "gromacs/utility/smalloc.h" #include "gromacs/utility/stringutil.h" @@ -74,7 +75,7 @@ bool buildSupportsNonbondedOnGpu(std::string* error) { errorReasons.emplace_back("double precision"); } - if (!c_binarySupportsGpus) + if (!GMX_GPU) { errorReasons.emplace_back("non-GPU build of GROMACS"); } diff --git a/src/gromacs/gpu_utils/gpu_utils.cu b/src/gromacs/gpu_utils/gpu_utils.cu index e0ae3bed30..c68a8cda63 100644 --- a/src/gromacs/gpu_utils/gpu_utils.cu +++ b/src/gromacs/gpu_utils/gpu_utils.cu @@ -53,7 +53,8 @@ #include "gromacs/gpu_utils/device_context.h" #include "gromacs/gpu_utils/device_stream.h" #include "gromacs/gpu_utils/pmalloc_cuda.h" -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_information.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/utility/basedefinitions.h" #include "gromacs/utility/cstringutil.h" #include "gromacs/utility/exceptions.h" diff --git a/src/gromacs/gpu_utils/gputraits.cuh b/src/gromacs/gpu_utils/gputraits.cuh index a165df595d..fec113b4b4 100644 --- a/src/gromacs/gpu_utils/gputraits.cuh +++ b/src/gromacs/gpu_utils/gputraits.cuh @@ -46,8 +46,6 @@ */ #include -#include "gromacs/hardware/gpu_hw_info.h" - //! Device texture for fast read-only data fetching using DeviceTexture = cudaTextureObject_t; diff --git a/src/gromacs/gpu_utils/gputraits_ocl.h b/src/gromacs/gpu_utils/gputraits_ocl.h index b3c6c8340e..489bb0527c 100644 --- a/src/gromacs/gpu_utils/gputraits_ocl.h +++ b/src/gromacs/gpu_utils/gputraits_ocl.h @@ -46,7 +46,6 @@ */ #include "gromacs/gpu_utils/gmxopencl.h" -#include "gromacs/hardware/gpu_hw_info.h" using DeviceTexture = void*; diff --git a/src/gromacs/gpu_utils/oclutils.h b/src/gromacs/gpu_utils/oclutils.h index bb776d4781..dca575367a 100644 --- a/src/gromacs/gpu_utils/oclutils.h +++ b/src/gromacs/gpu_utils/oclutils.h @@ -50,6 +50,7 @@ #include "gromacs/gpu_utils/gputraits_ocl.h" #include "gromacs/utility/exceptions.h" #include "gromacs/utility/gmxassert.h" +#include "gromacs/utility/stringutil.h" enum class GpuApiCallBehavior; diff --git a/src/gromacs/gpu_utils/tests/device_stream_manager.cpp b/src/gromacs/gpu_utils/tests/device_stream_manager.cpp index 1491669201..e3db2cb19e 100644 --- a/src/gromacs/gpu_utils/tests/device_stream_manager.cpp +++ b/src/gromacs/gpu_utils/tests/device_stream_manager.cpp @@ -108,8 +108,7 @@ TEST_F(DeviceStreamManagerTest, CorrectStreamsAreReturnedOnNonbondedDevice) // that we've called, so it is not very useful. const bool useTiming = false; - // TODO Is it enough to only test one device? - for (const auto* deviceInfo : getDeviceInfos()) + for (const auto& deviceInfo : getDeviceInfoList()) { EXPECT_FALSE(deviceInfo == nullptr) << "Device information should be provided for the GPU builds."; diff --git a/src/gromacs/gpu_utils/tests/devicetransfers.cpp b/src/gromacs/gpu_utils/tests/devicetransfers.cpp index 5ea7aadbaa..9a71a58730 100644 --- a/src/gromacs/gpu_utils/tests/devicetransfers.cpp +++ b/src/gromacs/gpu_utils/tests/devicetransfers.cpp @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2017,2019, by the GROMACS development team, led by + * Copyright (c) 2017,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -49,7 +49,9 @@ namespace gmx { -void doDeviceTransfers(const gmx_gpu_info_t& /*gpuInfo*/, ArrayRef input, ArrayRef output) +void doDeviceTransfers(const DeviceInformation& /* deviceInfo */, + ArrayRef input, + ArrayRef output) { GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size"); // We can't have any valid GPUs for this build configuration. diff --git a/src/gromacs/gpu_utils/tests/devicetransfers.cu b/src/gromacs/gpu_utils/tests/devicetransfers.cu index 0636285a1e..4e7e14779d 100644 --- a/src/gromacs/gpu_utils/tests/devicetransfers.cu +++ b/src/gromacs/gpu_utils/tests/devicetransfers.cu @@ -49,8 +49,7 @@ #include "devicetransfers.h" #include "gromacs/gpu_utils/cudautils.cuh" -#include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_information.h" #include "gromacs/utility/arrayref.h" #include "gromacs/utility/exceptions.h" #include "gromacs/utility/gmxassert.h" @@ -76,23 +75,16 @@ static void throwUponFailure(cudaError_t status, const char* message) } // namespace -void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef input, ArrayRef output) +void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef input, ArrayRef output) { GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size"); - const auto compatibleGpus = getCompatibleGpus(gpuInfo); - if (compatibleGpus.empty()) - { - std::copy(input.begin(), input.end(), output.begin()); - return; - } cudaError_t status; - const auto* device = getDeviceInfo(gpuInfo, compatibleGpus[0]); - int oldDeviceId; + int oldDeviceId; status = cudaGetDevice(&oldDeviceId); throwUponFailure(status, "getting old device id"); - status = cudaSetDevice(device->id); + status = cudaSetDevice(deviceInfo.id); throwUponFailure(status, "setting device id to the first compatible GPU"); void* devicePointer; diff --git a/src/gromacs/gpu_utils/tests/devicetransfers.h b/src/gromacs/gpu_utils/tests/devicetransfers.h index 64acdd5b0a..1315741de5 100644 --- a/src/gromacs/gpu_utils/tests/devicetransfers.h +++ b/src/gromacs/gpu_utils/tests/devicetransfers.h @@ -51,7 +51,7 @@ #ifndef GMX_GPU_UTILS_TESTS_DEVICETRANSFERS_H #define GMX_GPU_UTILS_TESTS_DEVICETRANSFERS_H -struct gmx_gpu_info_t; +struct DeviceInformation; namespace gmx { @@ -65,7 +65,7 @@ class ArrayRef; * do a simple host-side buffer copy instead. * * \throws InternalError Upon any GPU API error condition. */ -void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef input, ArrayRef output); +void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef input, ArrayRef output); } // namespace gmx diff --git a/src/gromacs/gpu_utils/tests/devicetransfers_ocl.cpp b/src/gromacs/gpu_utils/tests/devicetransfers_ocl.cpp index 8338e58fa8..012700123b 100644 --- a/src/gromacs/gpu_utils/tests/devicetransfers_ocl.cpp +++ b/src/gromacs/gpu_utils/tests/devicetransfers_ocl.cpp @@ -42,8 +42,8 @@ #include "gromacs/gpu_utils/gmxopencl.h" #include "gromacs/gpu_utils/oclutils.h" +#include "gromacs/hardware/device_information.h" #include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/gpu_hw_info.h" #include "gromacs/utility/arrayref.h" #include "gromacs/utility/exceptions.h" #include "gromacs/utility/gmxassert.h" @@ -71,24 +71,18 @@ void throwUponFailure(cl_int status, const char* message) } // namespace -void doDeviceTransfers(const gmx_gpu_info_t& gpuInfo, ArrayRef input, ArrayRef output) +void doDeviceTransfers(const DeviceInformation& deviceInfo, ArrayRef input, ArrayRef output) { GMX_RELEASE_ASSERT(input.size() == output.size(), "Input and output must have matching size"); - const auto compatibleGpus = getCompatibleGpus(gpuInfo); - if (compatibleGpus.empty()) - { - std::copy(input.begin(), input.end(), output.begin()); - return; - } + cl_int status; - const auto* device = getDeviceInfo(gpuInfo, compatibleGpus[0]); cl_context_properties properties[] = { - CL_CONTEXT_PLATFORM, reinterpret_cast(device->oclPlatformId), 0 + CL_CONTEXT_PLATFORM, reinterpret_cast(deviceInfo.oclPlatformId), 0 }; // Give uncrustify more space - auto deviceId = device->oclDeviceId; + auto deviceId = deviceInfo.oclDeviceId; auto context = clCreateContext(properties, 1, &deviceId, nullptr, nullptr, &status); throwUponFailure(status, "creating context"); auto commandQueue = clCreateCommandQueue(context, deviceId, 0, &status); diff --git a/src/gromacs/gpu_utils/tests/gputest.cpp b/src/gromacs/gpu_utils/tests/gputest.cpp index 4caabc374f..e89581fb6d 100644 --- a/src/gromacs/gpu_utils/tests/gputest.cpp +++ b/src/gromacs/gpu_utils/tests/gputest.cpp @@ -44,8 +44,9 @@ #include +#include "gromacs/gpu_utils/gpu_utils.h" +#include "gromacs/hardware/device_information.h" #include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/gpu_hw_info.h" #include "gromacs/utility/smalloc.h" namespace gmx @@ -55,35 +56,18 @@ namespace test GpuTest::GpuTest() { - snew(gpuInfo_, 1); - if (isGpuDetectionFunctional(nullptr)) + if (canPerformDeviceDetection(nullptr)) { - findGpus(gpuInfo_); - compatibleGpuIds_ = getCompatibleGpus(*gpuInfo_); + deviceInfoList_ = findDevices(); } // Failing to find valid GPUs does not require further action } -GpuTest::~GpuTest() -{ - free_gpu_info(gpuInfo_); - sfree(gpuInfo_); -} +GpuTest::~GpuTest() = default; -bool GpuTest::haveCompatibleGpus() const +std::vector>& GpuTest::getDeviceInfoList() { - return !compatibleGpuIds_.empty(); -} - -std::vector GpuTest::getDeviceInfos() const -{ - std::vector deviceInfos; - deviceInfos.reserve(compatibleGpuIds_.size()); - for (const auto& id : compatibleGpuIds_) - { - deviceInfos.emplace_back(getDeviceInfo(*gpuInfo_, id)); - } - return deviceInfos; + return deviceInfoList_; } } // namespace test diff --git a/src/gromacs/gpu_utils/tests/gputest.h b/src/gromacs/gpu_utils/tests/gputest.h index a78b00defe..46a53a4f05 100644 --- a/src/gromacs/gpu_utils/tests/gputest.h +++ b/src/gromacs/gpu_utils/tests/gputest.h @@ -45,8 +45,9 @@ #include +#include "gromacs/hardware/device_management.h" + struct DeviceInformation; -struct gmx_gpu_info_t; namespace gmx { @@ -56,17 +57,13 @@ namespace test class GpuTest : public ::testing::Test { public: - //! Information about GPUs that are present. - gmx_gpu_info_t* gpuInfo_; - //! Contains the IDs of all compatible GPUs - std::vector compatibleGpuIds_; + //! List of all available devices + std::vector> deviceInfoList_; GpuTest(); ~GpuTest() override; - //! Return whether compatible GPUs were found - bool haveCompatibleGpus() const; //! Return a vector of handles, each to a device info for a compatible GPU. - std::vector getDeviceInfos() const; + std::vector>& getDeviceInfoList(); }; } // namespace test diff --git a/src/gromacs/gpu_utils/tests/hostallocator.cpp b/src/gromacs/gpu_utils/tests/hostallocator.cpp index 2817628b9e..7f2408fb44 100644 --- a/src/gromacs/gpu_utils/tests/hostallocator.cpp +++ b/src/gromacs/gpu_utils/tests/hostallocator.cpp @@ -97,7 +97,7 @@ ArrayRef charArrayRefFromArray(T* data, size_t size) //! Does a device transfer of \c input to the device in \c gpuInfo, and back to \c output. template -void runTest(const gmx_gpu_info_t& gpuInfo, ArrayRef input, ArrayRef output) +void runTest(const DeviceInformation& deviceInfo, ArrayRef input, ArrayRef output) { // Convert the views of input and output to flat non-const chars, // so that there's no templating when we call doDeviceTransfers. @@ -105,7 +105,8 @@ void runTest(const gmx_gpu_info_t& gpuInfo, ArrayRef input, ArrayRef outpu auto outputRef = charArrayRefFromArray(output.data(), output.size()); ASSERT_EQ(inputRef.size(), outputRef.size()); - doDeviceTransfers(gpuInfo, inputRef, outputRef); + + doDeviceTransfers(deviceInfo, inputRef, outputRef); compareViews(input, output); } @@ -198,12 +199,15 @@ TYPED_TEST(HostAllocatorTestCopyable, VectorsWithDefaultHostAllocatorAlwaysWorks TYPED_TEST(HostAllocatorTestCopyable, TransfersWithoutPinningWork) { - typename TestFixture::VectorType input; - fillInput(&input, 1); - typename TestFixture::VectorType output; - output.resizeWithPadding(input.size()); + for (const DeviceInformation& compatibleDeviceInfo : getCompatibleDevices(this->deviceInfoList_)) + { + typename TestFixture::VectorType input; + fillInput(&input, 1); + typename TestFixture::VectorType output; + output.resizeWithPadding(input.size()); - runTest(*this->gpuInfo_, makeArrayRef(input), makeArrayRef(output)); + runTest(compatibleDeviceInfo, makeArrayRef(input), makeArrayRef(output)); + } } TYPED_TEST(HostAllocatorTestCopyable, FillInputAlsoWorksAfterCallingReserve) @@ -292,19 +296,17 @@ TYPED_TEST(HostAllocatorTestNoMem, Comparison) TYPED_TEST(HostAllocatorTestCopyable, TransfersWithPinningWorkWithCuda) { - if (!this->haveCompatibleGpus()) + for (auto& deviceInfo : this->deviceInfoList_) { - return; + typename TestFixture::VectorType input; + changePinningPolicy(&input, PinningPolicy::PinnedIfSupported); + fillInput(&input, 1); + typename TestFixture::VectorType output; + changePinningPolicy(&output, PinningPolicy::PinnedIfSupported); + output.resizeWithPadding(input.size()); + + runTest(*deviceInfo, makeArrayRef(input), makeArrayRef(output)); } - - typename TestFixture::VectorType input; - changePinningPolicy(&input, PinningPolicy::PinnedIfSupported); - fillInput(&input, 1); - typename TestFixture::VectorType output; - changePinningPolicy(&output, PinningPolicy::PinnedIfSupported); - output.resizeWithPadding(input.size()); - - runTest(*this->gpuInfo_, makeArrayRef(input), makeArrayRef(output)); } //! Helper function for wrapping a call to isHostMemoryPinned. @@ -317,7 +319,7 @@ bool isPinned(const VectorType& v) TYPED_TEST(HostAllocatorTestCopyable, ManualPinningOperationsWorkWithCuda) { - if (!this->haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } diff --git a/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp b/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp index 4317dfdbb7..9c2ae73c7c 100644 --- a/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp +++ b/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp @@ -70,7 +70,7 @@ using PinnedMemoryCheckerTest = GpuTest; TEST_F(PinnedMemoryCheckerTest, DefaultContainerIsRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } @@ -81,7 +81,7 @@ TEST_F(PinnedMemoryCheckerTest, DefaultContainerIsRecognized) TEST_F(PinnedMemoryCheckerTest, NonpinnedContainerIsRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } @@ -93,7 +93,7 @@ TEST_F(PinnedMemoryCheckerTest, NonpinnedContainerIsRecognized) TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } @@ -105,7 +105,7 @@ TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized) TEST_F(PinnedMemoryCheckerTest, PinningChangesAreRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } @@ -121,7 +121,7 @@ TEST_F(PinnedMemoryCheckerTest, PinningChangesAreRecognized) TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } @@ -134,7 +134,7 @@ TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized) TEST_F(PinnedMemoryCheckerTest, PinnedCBufferIsRecognized) { - if (!haveCompatibleGpus()) + if (!canComputeOnDevice()) { return; } diff --git a/src/gromacs/gpu_utils/tests/typecasts.cpp b/src/gromacs/gpu_utils/tests/typecasts.cpp index 319813828b..0246d4f106 100644 --- a/src/gromacs/gpu_utils/tests/typecasts.cpp +++ b/src/gromacs/gpu_utils/tests/typecasts.cpp @@ -48,7 +48,7 @@ # include -# include "gromacs/gpu_utils/gpu_testutils.h" +# include "gromacs/hardware/device_management.h" # include "gromacs/utility/exceptions.h" # include "testutils/testasserts.h" @@ -74,7 +74,7 @@ TEST(GpuDataTypesCompatibilityTest, RVecAndFloat3OnHost) TEST(GpuDataTypesCompatibilityTest, RVecAndFloat3OnDevice) { - if (canComputeOnGpu()) + if (canComputeOnDevice()) { std::vector rVecOutput(rVecInput.size()); convertRVecToFloat3OnDevice(rVecOutput, rVecInput); diff --git a/src/gromacs/hardware/detecthardware.cpp b/src/gromacs/hardware/detecthardware.cpp index 7e8ac92c24..65011de939 100644 --- a/src/gromacs/hardware/detecthardware.cpp +++ b/src/gromacs/hardware/detecthardware.cpp @@ -60,11 +60,13 @@ #include "gromacs/utility/exceptions.h" #include "gromacs/utility/fatalerror.h" #include "gromacs/utility/gmxassert.h" +#include "gromacs/utility/inmemoryserializer.h" #include "gromacs/utility/logger.h" #include "gromacs/utility/mutex.h" #include "gromacs/utility/physicalnodecommunicator.h" #include "architecture.h" +#include "device_information.h" #ifdef HAVE_UNISTD_H # include // sysconf() @@ -77,10 +79,7 @@ gmx_hw_info_t::gmx_hw_info_t(std::unique_ptr cpuInfo, { } -gmx_hw_info_t::~gmx_hw_info_t() -{ - free_gpu_info(&gpu_info); -} +gmx_hw_info_t::~gmx_hw_info_t() = default; namespace gmx { @@ -112,10 +111,16 @@ static void gmx_detect_gpus(const gmx::MDLogger& mdlog, const PhysicalNodeCommunicator& physicalNodeComm, compat::not_null hardwareInfo) { - hardwareInfo->gpu_info.bDetectGPUs = canPerformGpuDetection(); - - if (!hardwareInfo->gpu_info.bDetectGPUs) + std::string errorMessage; + if (!canPerformDeviceDetection(&errorMessage)) { + GMX_LOG(mdlog.info) + .asParagraph() + .appendTextFormatted( + "NOTE: Detection of GPUs failed. The API reported:\n" + " %s\n" + " GROMACS cannot run tasks on a GPU.", + errorMessage.c_str()); return; } @@ -137,7 +142,7 @@ static void gmx_detect_gpus(const gmx::MDLogger& mdlog, if (isMasterRankOfPhysicalNode || allRanksMustDetectGpus) { std::string errorMessage; - gpusCanBeDetected = isGpuDetectionFunctional(&errorMessage); + gpusCanBeDetected = isDeviceDetectionFunctional(&errorMessage); if (!gpusCanBeDetected) { GMX_LOG(mdlog.info) @@ -152,30 +157,22 @@ static void gmx_detect_gpus(const gmx::MDLogger& mdlog, if (gpusCanBeDetected) { - findGpus(&hardwareInfo->gpu_info); + hardwareInfo->deviceInfoList = findDevices(); // No need to tell the user anything at this point, they get a // hardware report later. } #if GMX_LIB_MPI - if (!allRanksMustDetectGpus) + if (!allRanksMustDetectGpus && !hardwareInfo->deviceInfoList.empty()) { - /* Broadcast the GPU info to the other ranks within this node */ - MPI_Bcast(&hardwareInfo->gpu_info.n_dev, 1, MPI_INT, 0, physicalNodeComm.comm_); - - if (hardwareInfo->gpu_info.n_dev > 0) - { - int dev_size; + gmx::InMemorySerializer writer; + serializeDeviceInformations(hardwareInfo->deviceInfoList, &writer); + auto buffer = writer.finishAndGetBuffer(); - dev_size = hardwareInfo->gpu_info.n_dev * sizeof_gpu_dev_info(); + MPI_Bcast(buffer.data(), buffer.size(), MPI_BYTE, 0, physicalNodeComm.comm_); - if (!isMasterRankOfPhysicalNode) - { - hardwareInfo->gpu_info.deviceInfo = (struct DeviceInformation*)malloc(dev_size); - } - MPI_Bcast(hardwareInfo->gpu_info.deviceInfo, dev_size, MPI_BYTE, 0, physicalNodeComm.comm_); - MPI_Bcast(&hardwareInfo->gpu_info.n_dev_compatible, 1, MPI_INT, 0, physicalNodeComm.comm_); - } + gmx::InMemoryDeserializer reader(buffer, false); + hardwareInfo->deviceInfoList = deserializeDeviceInformations(&writer); } #endif } @@ -194,28 +191,27 @@ static void gmx_collect_hardware_mpi(const gmx::CpuInfo& cpuInfo, && (cpuInfo.model() == 1 || cpuInfo.model() == 17 || cpuInfo.model() == 8 || cpuInfo.model() == 24)) || cpuInfo.vendor() == CpuInfo::Vendor::Hygon); + + int numCompatibleDevices = getCompatibleDevices(hardwareInfo->deviceInfoList).size(); #if GMX_LIB_MPI - int nhwthread, ngpu, i; + int nhwthread; int gpu_hash; nhwthread = hardwareInfo->nthreads_hw_avail; - ngpu = hardwareInfo->gpu_info.n_dev_compatible; /* Create a unique hash of the GPU type(s) in this node */ gpu_hash = 0; /* Here it might be better to only loop over the compatible GPU, but we * don't have that information available and it would also require * removing the device ID from the device info string. */ - for (i = 0; i < hardwareInfo->gpu_info.n_dev; i++) + for (const auto& deviceInfo : hardwareInfo->deviceInfoList) { - char stmp[STRLEN]; - /* Since the device ID is incorporated in the hash, the order of * the GPUs affects the hash. Also two identical GPUs won't give * a gpu_hash of zero after XORing. */ - get_gpu_device_info_string(stmp, hardwareInfo->gpu_info, i); - gpu_hash ^= gmx_string_fullhash_func(stmp, gmx_string_hash_init); + std::string deviceInfoString = getDeviceInformationString(*deviceInfo); + gpu_hash ^= gmx_string_fullhash_func(deviceInfoString.c_str(), gmx_string_hash_init); } constexpr int numElementsCounts = 4; @@ -230,7 +226,7 @@ static void gmx_collect_hardware_mpi(const gmx::CpuInfo& cpuInfo, countsLocal[0] = 1; countsLocal[1] = ncore; countsLocal[2] = nhwthread; - countsLocal[3] = ngpu; + countsLocal[3] = numCompatibleDevices; } MPI_Allreduce(countsLocal.data(), countsReduced.data(), countsLocal.size(), MPI_INT, @@ -246,7 +242,7 @@ static void gmx_collect_hardware_mpi(const gmx::CpuInfo& cpuInfo, */ maxMinLocal[0] = ncore; maxMinLocal[1] = nhwthread; - maxMinLocal[2] = ngpu; + maxMinLocal[2] = numCompatibleDevices; maxMinLocal[3] = static_cast(gmx::simdSuggested(cpuInfo)); maxMinLocal[4] = gpu_hash; maxMinLocal[5] = -maxMinLocal[0]; @@ -283,9 +279,9 @@ static void gmx_collect_hardware_mpi(const gmx::CpuInfo& cpuInfo, hardwareInfo->nhwthread_tot = hardwareInfo->nthreads_hw_avail; hardwareInfo->nhwthread_min = hardwareInfo->nthreads_hw_avail; hardwareInfo->nhwthread_max = hardwareInfo->nthreads_hw_avail; - hardwareInfo->ngpu_compatible_tot = hardwareInfo->gpu_info.n_dev_compatible; - hardwareInfo->ngpu_compatible_min = hardwareInfo->gpu_info.n_dev_compatible; - hardwareInfo->ngpu_compatible_max = hardwareInfo->gpu_info.n_dev_compatible; + hardwareInfo->ngpu_compatible_tot = numCompatibleDevices; + hardwareInfo->ngpu_compatible_min = numCompatibleDevices; + hardwareInfo->ngpu_compatible_max = numCompatibleDevices; hardwareInfo->simd_suggest_min = static_cast(simdSuggested(cpuInfo)); hardwareInfo->simd_suggest_max = static_cast(simdSuggested(cpuInfo)); hardwareInfo->bIdenticalGPUs = TRUE; @@ -460,10 +456,6 @@ gmx_hw_info_t* gmx_detect_hardware(const gmx::MDLogger& mdlog, const PhysicalNod hardwareInfo->nthreads_hw_avail = hardwareInfo->hardwareTopology->machine().logicalProcessorCount; // Detect GPUs - hardwareInfo->gpu_info.n_dev = 0; - hardwareInfo->gpu_info.n_dev_compatible = 0; - hardwareInfo->gpu_info.deviceInfo = nullptr; - gmx_detect_gpus(mdlog, physicalNodeComm, compat::make_not_null(hardwareInfo)); gmx_collect_hardware_mpi(*hardwareInfo->cpuInfo, physicalNodeComm, compat::make_not_null(hardwareInfo)); @@ -474,9 +466,4 @@ gmx_hw_info_t* gmx_detect_hardware(const gmx::MDLogger& mdlog, const PhysicalNod return g_hardwareInfo.get(); } -bool compatibleGpusFound(const gmx_gpu_info_t& gpu_info) -{ - return gpu_info.n_dev_compatible > 0; -} - } // namespace gmx diff --git a/src/gromacs/hardware/detecthardware.h b/src/gromacs/hardware/detecthardware.h index 0de2d34ac6..efacba0028 100644 --- a/src/gromacs/hardware/detecthardware.h +++ b/src/gromacs/hardware/detecthardware.h @@ -36,7 +36,6 @@ #ifndef GMX_HARDWARE_DETECTHARDWARE_H #define GMX_HARDWARE_DETECTHARDWARE_H -struct gmx_gpu_info_t; struct gmx_hw_info_t; namespace gmx @@ -62,9 +61,6 @@ class PhysicalNodeCommunicator; gmx_hw_info_t* gmx_detect_hardware(const gmx::MDLogger& mdlog, const PhysicalNodeCommunicator& physicalNodeComm); -//! Return whether compatible GPUs were found. -bool compatibleGpusFound(const gmx_gpu_info_t& gpu_info); - } // namespace gmx #endif diff --git a/src/gromacs/hardware/device_information.h b/src/gromacs/hardware/device_information.h index d9116a3a72..8c8020efaa 100644 --- a/src/gromacs/hardware/device_information.h +++ b/src/gromacs/hardware/device_information.h @@ -1,7 +1,8 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -33,13 +34,14 @@ * the research papers on the package. Check out http://www.gromacs.org. */ /*! \libinternal \file - * \brief Declares the GPU type traits for non-GPU builds. + * \brief Declares the GPU information structure and its helpers * + * \author Anca Hamuraru + * \author Dimitrios Karkoulis + * \author Teemu Virolainen * \author Mark Abraham + * \author Szilárd Páll * \author Artem Zhmurov - * - * \inlibraryapi - * \ingroup module_hardware */ #ifndef GMX_HARDWARE_DEVICE_INFORMATION_H #define GMX_HARDWARE_DEVICE_INFORMATION_H @@ -124,7 +126,7 @@ enum class DeviceVendor : int struct DeviceInformation { //! Device status. - DeviceStatus stat; + DeviceStatus status; //! ID of the device. int id; diff --git a/src/gromacs/hardware/device_management.cpp b/src/gromacs/hardware/device_management.cpp index 1d03f1b0be..873d1258c1 100644 --- a/src/gromacs/hardware/device_management.cpp +++ b/src/gromacs/hardware/device_management.cpp @@ -1,8 +1,8 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team. - * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -36,6 +36,11 @@ /*! \internal \file * \brief Defines the CPU stubs for the device management. * + * \author Anca Hamuraru + * \author Dimitrios Karkoulis + * \author Teemu Virolainen + * \author Mark Abraham + * \author Szilárd Páll * \author Artem Zhmurov * * \ingroup module_hardware @@ -44,47 +49,26 @@ #include "device_management.h" -bool isGpuDetectionFunctional(std::string* errorMessage) -{ - if (errorMessage != nullptr) - { - errorMessage->assign("GROMACS has been built without GPU support."); - } - return false; -} +#include "gromacs/gpu_utils/gputraits.h" +#include "gromacs/utility/fatalerror.h" -void findGpus(gmx_gpu_info_t* /* gpu_info */) -{ - GMX_RELEASE_ASSERT(false, "Trying to initialize GPUs in the build that does not support them."); -} +#include "device_information.h" -void init_gpu(const DeviceInformation* /* deviceInfo */) +std::vector> findDevices() { - GMX_RELEASE_ASSERT(false, "Trying to initialize GPU in the build that does not support GPUs."); + return {}; } -void free_gpu(const DeviceInformation* /* deviceInfo */) {} +void setActiveDevice(const DeviceInformation& /* deviceInfo */) {} -DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& /* gpu_info */, int /* deviceId */) -{ - GMX_RELEASE_ASSERT( - false, "Trying to get GPU device information in the build that does not support GPUs."); - return nullptr; -} +void releaseDevice(DeviceInformation* /* deviceInfo */) {} -void get_gpu_device_info_string(char* /* s */, const gmx_gpu_info_t& /* gpu_info */, int /* index */) +std::string getDeviceInformationString(const DeviceInformation& /* deviceInfo */) { - GMX_RELEASE_ASSERT( - false, - "Trying to get the GPU device description in the build that does not support GPUs."); + gmx_fatal(FARGS, "Device information requested in CPU build."); } -size_t sizeof_gpu_dev_info() +bool isDeviceDetectionFunctional(std::string* /* errorMessage */) { - return 0; -} - -DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& /* gpu_info */, int /* index */) -{ - return DeviceStatus::Nonexistent; + return false; } diff --git a/src/gromacs/hardware/device_management.cu b/src/gromacs/hardware/device_management.cu index fba12ace11..32708873ec 100644 --- a/src/gromacs/hardware/device_management.cu +++ b/src/gromacs/hardware/device_management.cu @@ -1,8 +1,8 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team. - * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -36,6 +36,11 @@ /*! \internal \file * \brief Defines the CUDA implementations of the device management. * + * \author Anca Hamuraru + * \author Dimitrios Karkoulis + * \author Teemu Virolainen + * \author Mark Abraham + * \author Szilárd Páll * \author Artem Zhmurov * * \ingroup module_hardware @@ -49,23 +54,27 @@ #include "gromacs/gpu_utils/cudautils.cuh" #include "gromacs/gpu_utils/device_context.h" #include "gromacs/gpu_utils/device_stream.h" +#include "gromacs/utility/exceptions.h" #include "gromacs/utility/programcontext.h" #include "gromacs/utility/smalloc.h" +#include "gromacs/utility/stringutil.h" + +#include "device_information.h" /*! \internal \brief * Max number of devices supported by CUDA (for consistency checking). * * In reality it is 16 with CUDA <=v5.0, but let's stay on the safe side. */ -static int cuda_max_device_count = 32; +static int c_cudaMaxDeviceCount = 32; /** Dummy kernel used for sanity checking. */ -static __global__ void k_dummy_test(void) {} +static __global__ void dummy_kernel(void) {} static cudaError_t checkCompiledTargetCompatibility(int deviceId, const cudaDeviceProp& deviceProp) { cudaFuncAttributes attributes; - cudaError_t stat = cudaFuncGetAttributes(&attributes, k_dummy_test); + cudaError_t stat = cudaFuncGetAttributes(&attributes, dummy_kernel); if (cudaErrorInvalidDeviceFunction == stat) { @@ -120,7 +129,7 @@ static DeviceStatus isDeviceFunctional(int dev_id, const cudaDeviceProp& dev_pro } /* things might go horribly wrong if cudart is not compatible with the driver */ - if (dev_count < 0 || dev_count > cuda_max_device_count) + if (dev_count < 0 || dev_count > c_cudaMaxDeviceCount) { return DeviceStatus::NonFunctional; } @@ -186,11 +195,11 @@ static DeviceStatus isDeviceFunctional(int dev_id, const cudaDeviceProp& dev_pro { KernelLaunchConfig config; config.blockSize[0] = 512; - const auto dummyArguments = prepareGpuKernelArguments(k_dummy_test, config); + const auto dummyArguments = prepareGpuKernelArguments(dummy_kernel, config); DeviceInformation deviceInfo; const DeviceContext deviceContext(deviceInfo); const DeviceStream deviceStream(deviceContext, DeviceStreamPriority::Normal, false); - launchGpuKernel(k_dummy_test, config, deviceStream, nullptr, "Dummy kernel", dummyArguments); + launchGpuKernel(dummy_kernel, config, deviceStream, nullptr, "Dummy kernel", dummyArguments); } catch (gmx::GromacsException& ex) { @@ -216,16 +225,16 @@ static DeviceStatus isDeviceFunctional(int dev_id, const cudaDeviceProp& dev_pro return DeviceStatus::Compatible; } -/*! \brief Returns true if the gpu characterized by the device properties is - * supported by the native gpu acceleration. +/*! \brief Returns true if the gpu characterized by the device properties is supported + * by the native gpu acceleration. * - * \param[in] dev_prop the CUDA device properties of the gpus to test. - * \returns true if the GPU properties passed indicate a compatible - * GPU, otherwise false. + * \param[in] deviceProperties The CUDA device properties of the gpus to test. + * \returns True if the GPU properties passed indicate a compatible + * GPU, otherwise false. */ -static bool is_gmx_supported_gpu(const cudaDeviceProp& dev_prop) +static bool isDeviceGenerationSupported(const cudaDeviceProp& deviceProperties) { - return (dev_prop.major >= 3); + return (deviceProperties.major >= 3); } /*! \brief Checks if a GPU with a given ID is supported by the native GROMACS acceleration. @@ -244,14 +253,14 @@ static bool is_gmx_supported_gpu(const cudaDeviceProp& dev_prop) */ static DeviceStatus checkDeviceStatus(int deviceId, const cudaDeviceProp& deviceProp) { - if (!is_gmx_supported_gpu(deviceProp)) + if (!isDeviceGenerationSupported(deviceProp)) { return DeviceStatus::Incompatible; } return isDeviceFunctional(deviceId, deviceProp); } -bool isGpuDetectionFunctional(std::string* errorMessage) +bool isDeviceDetectionFunctional(std::string* errorMessage) { cudaError_t stat; int driverVersion = -1; @@ -306,27 +315,22 @@ bool isGpuDetectionFunctional(std::string* errorMessage) return true; } -void findGpus(gmx_gpu_info_t* gpu_info) +std::vector> findDevices() { - assert(gpu_info); - - gpu_info->n_dev_compatible = 0; - - int ndev; - cudaError_t stat = cudaGetDeviceCount(&ndev); + int numDevices; + cudaError_t stat = cudaGetDeviceCount(&numDevices); if (stat != cudaSuccess) { GMX_THROW(gmx::InternalError( - "Invalid call of findGpus() when CUDA API returned an error, perhaps " - "canDetectGpus() was not called appropriately beforehand.")); + "Invalid call of findDevices() when CUDA API returned an error, perhaps " + "canPerformDeviceDetection() was not called appropriately beforehand.")); } // We expect to start device support/sanity checks with a clean runtime error state gmx::ensureNoPendingCudaError(""); - DeviceInformation* devs; - snew(devs, ndev); - for (int i = 0; i < ndev; i++) + std::vector> deviceInfoList(numDevices); + for (int i = 0; i < numDevices; i++) { cudaDeviceProp prop; memset(&prop, 0, sizeof(cudaDeviceProp)); @@ -334,15 +338,13 @@ void findGpus(gmx_gpu_info_t* gpu_info) const DeviceStatus checkResult = (stat != cudaSuccess) ? DeviceStatus::NonFunctional : checkDeviceStatus(i, prop); - devs[i].id = i; - devs[i].prop = prop; - devs[i].stat = checkResult; + deviceInfoList[i] = std::make_unique(); - if (checkResult == DeviceStatus::Compatible) - { - gpu_info->n_dev_compatible++; - } - else + deviceInfoList[i]->id = i; + deviceInfoList[i]->prop = prop; + deviceInfoList[i]->status = checkResult; + + if (checkResult != DeviceStatus::Compatible) { // TODO: // - we inspect the CUDA API state to retrieve and record any @@ -358,7 +360,7 @@ void findGpus(gmx_gpu_info_t* gpu_info) if ((stat = cudaGetLastError()) != cudaSuccess) { gmx_warning("An error occurred while sanity checking device #%d; %s: %s", - devs[i].id, cudaGetErrorName(stat), cudaGetErrorString(stat)); + deviceInfoList[i]->id, cudaGetErrorName(stat), cudaGetErrorString(stat)); } } } @@ -370,97 +372,67 @@ void findGpus(gmx_gpu_info_t* gpu_info) cudaGetErrorName(stat), cudaGetErrorString(stat)) .c_str()); - gpu_info->n_dev = ndev; - gpu_info->deviceInfo = devs; + return deviceInfoList; } -void init_gpu(const DeviceInformation* deviceInfo) +void setActiveDevice(const DeviceInformation& deviceInfo) { + int deviceId = deviceInfo.id; cudaError_t stat; - assert(deviceInfo); - - stat = cudaSetDevice(deviceInfo->id); + stat = cudaSetDevice(deviceId); if (stat != cudaSuccess) { - auto message = gmx::formatString("Failed to initialize GPU #%d", deviceInfo->id); + auto message = gmx::formatString("Failed to initialize GPU #%d", deviceId); CU_RET_ERR(stat, message.c_str()); } if (debug) { - fprintf(stderr, "Initialized GPU ID #%d: %s\n", deviceInfo->id, deviceInfo->prop.name); + fprintf(stderr, "Initialized GPU ID #%d: %s\n", deviceId, deviceInfo.prop.name); } } -void free_gpu(const DeviceInformation* deviceInfo) +void releaseDevice(DeviceInformation* deviceInfo) { - // One should only attempt to clear the device context when - // it has been used, but currently the only way to know that a GPU // device was used is that deviceInfo will be non-null. - if (deviceInfo == nullptr) + if (deviceInfo != nullptr) { - return; - } - - cudaError_t stat; + cudaError_t stat; - if (debug) - { int gpuid; stat = cudaGetDevice(&gpuid); - CU_RET_ERR(stat, "cudaGetDevice failed"); - fprintf(stderr, "Cleaning up context on GPU ID #%d\n", gpuid); - } - - stat = cudaDeviceReset(); - if (stat != cudaSuccess) - { - gmx_warning("Failed to free GPU #%d: %s", deviceInfo->id, cudaGetErrorString(stat)); - } -} + if (stat == cudaSuccess) + { + if (debug) + { + fprintf(stderr, "Cleaning up context on GPU ID #%d\n", gpuid); + } -DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId) -{ - if (deviceId < 0 || deviceId >= gpu_info.n_dev) - { - gmx_incons("Invalid GPU deviceId requested"); + stat = cudaDeviceReset(); + if (stat != cudaSuccess) + { + gmx_warning("Failed to free GPU #%d: %s", gpuid, cudaGetErrorString(stat)); + } + } } - return &gpu_info.deviceInfo[deviceId]; } -void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index) +std::string getDeviceInformationString(const DeviceInformation& deviceInfo) { - assert(s); - - if (index < 0 && index >= gpu_info.n_dev) - { - return; - } - - DeviceInformation* dinfo = &gpu_info.deviceInfo[index]; - - bool bGpuExists = - (dinfo->stat != DeviceStatus::Nonexistent && dinfo->stat != DeviceStatus::NonFunctional); + bool gpuExists = (deviceInfo.status != DeviceStatus::Nonexistent + && deviceInfo.status != DeviceStatus::NonFunctional); - if (!bGpuExists) + if (!gpuExists) { - sprintf(s, "#%d: %s, stat: %s", dinfo->id, "N/A", c_deviceStateString[dinfo->stat]); + return gmx::formatString("#%d: %s, stat: %s", deviceInfo.id, "N/A", + c_deviceStateString[deviceInfo.status]); } else { - sprintf(s, "#%d: NVIDIA %s, compute cap.: %d.%d, ECC: %3s, stat: %s", dinfo->id, - dinfo->prop.name, dinfo->prop.major, dinfo->prop.minor, - dinfo->prop.ECCEnabled ? "yes" : " no", c_deviceStateString[dinfo->stat]); + return gmx::formatString("#%d: NVIDIA %s, compute cap.: %d.%d, ECC: %3s, stat: %s", + deviceInfo.id, deviceInfo.prop.name, deviceInfo.prop.major, + deviceInfo.prop.minor, deviceInfo.prop.ECCEnabled ? "yes" : " no", + c_deviceStateString[deviceInfo.status]); } } - -size_t sizeof_gpu_dev_info(void) -{ - return sizeof(DeviceInformation); -} - -DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index) -{ - return info.deviceInfo[index].stat; -} diff --git a/src/gromacs/hardware/device_management.h b/src/gromacs/hardware/device_management.h index ed86982c67..717f6b237c 100644 --- a/src/gromacs/hardware/device_management.h +++ b/src/gromacs/hardware/device_management.h @@ -1,7 +1,8 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2020, by the GROMACS development team, led by + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -33,10 +34,17 @@ * the research papers on the package. Check out http://www.gromacs.org. */ /*! \libinternal \file + * \brief Declares functions to manage GPU resources. * - * \brief Implements the device management for OpenCL. + * This has several implementations: one for each supported GPU platform, + * and a stub implementation if the build does not support GPUs. * - * \author Artem Zhmurov + * \author Anca Hamuraru + * \author Dimitrios Karkoulis + * \author Teemu Virolainen + * \author Mark Abraham + * \author Szilárd Páll + * \author Artem Zhmurov * * \inlibraryapi * \ingroup module_hardware @@ -44,28 +52,32 @@ #ifndef GMX_HARDWARE_DEVICE_MANAGEMENT_H #define GMX_HARDWARE_DEVICE_MANAGEMENT_H -#include "gmxpre.h" - +#include #include #include -#include "gromacs/hardware/device_information.h" +#include "gromacs/utility/basedefinitions.h" +#include "gromacs/utility/iserializer.h" struct DeviceInformation; -enum class DeviceStatus : int; -struct gmx_gpu_info_t; -/*! \brief Return whether GPUs can be detected +/*! \brief Return whether GPUs can be detected. + * + * Returns true when this is a build of GROMACS configured to support + * GPU usage, GPU detection is not disabled by \c GMX_DISABLE_GPU_DETECTION + * environment variable and a valid device driver, ICD, and/or runtime was + * detected. Does not throw. * - * Returns true when this is a build of \Gromacs configured to support - * GPU usage, GPU detection is not disabled by an environment variable - * and a valid device driver, ICD, and/or runtime was detected. - * Does not throw. */ -bool canPerformGpuDetection(); + * \param[out] errorMessage When returning false on a build configured with + * GPU support and non-nullptr was passed, + * the string contains a descriptive message about + * why GPUs cannot be detected. + */ +bool canPerformDeviceDetection(std::string* errorMessage); /*! \brief Return whether GPU detection is functioning correctly * - * Returns true when this is a build of \Gromacs configured to support + * Returns true when this is a build of GROMACS configured to support * GPU usage, and a valid device driver, ICD, and/or runtime was detected. * * This function is not intended to be called from build @@ -77,65 +89,63 @@ bool canPerformGpuDetection(); * the string contains a descriptive message about * why GPUs cannot be detected. * - * Does not throw. */ -bool isGpuDetectionFunctional(std::string* errorMessage); + * Does not throw. + */ +bool isDeviceDetectionFunctional(std::string* errorMessage); + +/*! \brief Checks if one can compute on the GPU + * + * \returns True if the build supports GPUs and there are at least one available. + */ +bool canComputeOnDevice(); /*! \brief Find all GPUs in the system. * * Will detect every GPU supported by the device driver in use. - * Must only be called if canPerformGpuDetection() has returned true. - * This routine also checks for the compatibility of each and fill the - * gpu_info->deviceInfo array with the required information on each the - * device: ID, device properties, status. + * Must only be called if \c canPerformDeviceDetection() has returned true. + * This routine also checks for the compatibility of each device and fill the + * deviceInfo array with the required information on each device: ID, device + * properties, status. * * Note that this function leaves the GPU runtime API error state clean; * this is implemented ATM in the CUDA flavor. - * TODO: check if errors do propagate in OpenCL as they do in CUDA and - * whether there is a mechanism to "clear" them. * - * \param[in] gpu_info pointer to structure holding GPU information. + * \todo: Check if errors do propagate in OpenCL as they do in CUDA and + * whether there is a mechanism to "clear" them. + * + * \return Standard vector with the list of devices found * - * \throws InternalError if a GPU API returns an unexpected failure (because - * the call to canDetectGpus() should always prevent this occuring) + * \throws InternalError if a GPU API returns an unexpected failure (because + * the call to canDetectGpus() should always prevent this occuring) */ -void findGpus(gmx_gpu_info_t* gpu_info); +std::vector> findDevices(); -/*! \brief Return a container of the detected GPUs that are compatible. +/*! \brief Return a container of the detected GPU ids that are compatible. * * This function filters the result of the detection for compatible * GPUs, based on the previously run compatibility tests. * - * \param[in] gpu_info Information detected about GPUs, including compatibility. - * \return vector of IDs of GPUs already recorded as compatible */ -std::vector getCompatibleGpus(const gmx_gpu_info_t& gpu_info); - -/*! \brief Return a string describing how compatible the GPU with given \c index is. - * - * \param[in] gpu_info Information about detected GPUs - * \param[in] index index of GPU to ask about - * \returns A null-terminated C string describing the compatibility status, useful for error messages. - */ -const char* getGpuCompatibilityDescription(const gmx_gpu_info_t& gpu_info, int index); - -/*! \brief Frees the gpu_dev and dev_use array fields of \p gpu_info. + * \param[in] deviceInfoList An information on available devices. * - * \param[in] gpu_info pointer to structure holding GPU information + * \return Vector of DeviceInformations on GPUs recorded as compatible */ -void free_gpu_info(const gmx_gpu_info_t* gpu_info); +std::vector> +getCompatibleDevices(const std::vector>& deviceInfoList); -/*! \brief Initializes the GPU described by \c deviceInfo. +/*! \brief Set the active GPU. * - * TODO Doxygen complains about these - probably a Doxygen bug, since - * the patterns here are the same as elsewhere in this header. + * This sets the device for which the device information is passed active. Essential in CUDA, where + * the device buffers and kernel launches are not connected to the device context. In OpenCL, checks + * the device vendor and makes vendor-specific performance adjustments. * - * \param[in] deviceInfo device info of the GPU to initialize + * \param[in] deviceInfo Information on the device to be set. * * Issues a fatal error for any critical errors that occur during * initialization. */ -void init_gpu(const DeviceInformation* deviceInfo); +void setActiveDevice(const DeviceInformation& deviceInfo); -/*! \brief Frees up the CUDA GPU used by the active context at the time of calling. +/*! \brief Releases the GPU device used by the active context at the time of calling (CUDA only). * * If \c deviceInfo is nullptr, then it is understood that no device * was selected so no context is active to be freed. Otherwise, the @@ -144,45 +154,50 @@ void init_gpu(const DeviceInformation* deviceInfo); * required anymore, because subsequent attempts to free memory * associated with the context will otherwise fail. * - * Calls gmx_warning upon errors. + * Calls \c gmx_warning upon errors. * - * \param[in] deviceInfo device info of the GPU to clean up for + * \todo This should go through all the devices, not only the one currently active. + * Reseting only one device will not work, e.g. in CUDA tests. * - * \returns true if no error occurs during the freeing. + * \param[in] deviceInfo Information on the device to be released. */ -void free_gpu(const DeviceInformation* deviceInfo); +void releaseDevice(DeviceInformation* deviceInfo); -/*! \brief Return a pointer to the device info for \c deviceId +/*! \brief Formats and returns a device information string for a given GPU. + * + * Given an index *directly* into the array of available GPUs, returns + * a formatted info string for the respective GPU which includes ID, name, + * compute capability, and detection status. * - * \param[in] gpu_info GPU info of all detected devices in the system. - * \param[in] deviceId ID for the GPU device requested. + * \param[in] deviceInfo An information on device that is to be set. * - * \returns Pointer to the device info for \c deviceId. + * \returns A string describing the device. */ -DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId); +std::string getDeviceInformationString(const DeviceInformation& deviceInfo); -/*! \brief Formats and returns a device information string for a given GPU. - * - * Given an index *directly* into the array of available GPUs (gpu_dev) - * returns a formatted info string for the respective GPU which includes - * ID, name, compute capability, and detection status. +/*! \brief Return a string describing how compatible the GPU with given \c deviceId is. * - * \param[out] s pointer to output string (has to be allocated externally) - * \param[in] gpu_info Information about detected GPUs - * \param[in] index an index *directly* into the array of available GPUs + * \param[in] deviceInfoList An information on available devices. + * \param[in] deviceId An index of the device to check + * \returns A string describing the compatibility status, useful for error messages. */ -void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index); +std::string getDeviceCompatibilityDescription(const std::vector>& deviceInfoList, + int deviceId); +/*! \brief Serialization of information on devices for MPI broadcasting. + * + * \param[in] deviceInfoList The vector with device informations to serialize. + * \param[in] serializer Serializing object. + */ +void serializeDeviceInformations(const std::vector>& deviceInfoList, + gmx::ISerializer* serializer); -/*! \brief Returns the size of the gpu_dev_info struct. +/*! \brief Deserialization of information on devices after MPI broadcasting. * - * The size of gpu_dev_info can be used for allocation and communication. + * \param[in] serializer Serializing object. * - * \returns size in bytes of gpu_dev_info + * \return deviceInfoList Deserialized vector with device informations. */ -size_t sizeof_gpu_dev_info(); - -//! Get status of device with specified index -DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index); +std::vector> deserializeDeviceInformations(gmx::ISerializer* serializer); #endif // GMX_HARDWARE_DEVICE_MANAGEMENT_H diff --git a/src/gromacs/hardware/device_management_common.cpp b/src/gromacs/hardware/device_management_common.cpp index d5325b77e0..ad85eb05a6 100644 --- a/src/gromacs/hardware/device_management_common.cpp +++ b/src/gromacs/hardware/device_management_common.cpp @@ -1,8 +1,8 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team. - * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -34,26 +34,30 @@ * the research papers on the package. Check out http://www.gromacs.org. */ /*! \internal \file - * \brief Defines the implementations of the device management that are common for CPU, CUDA and OpenCL. + * \brief Defines the implementations of device management functions that + * are common for CPU, CUDA and OpenCL. * + * \author Anca Hamuraru + * \author Dimitrios Karkoulis + * \author Teemu Virolainen + * \author Mark Abraham + * \author Szilárd Páll * \author Artem Zhmurov * * \ingroup module_hardware */ #include "gmxpre.h" -#include - -#include "gromacs/hardware/device_information.h" #include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/gpu_hw_info.h" -#include "gromacs/utility/smalloc.h" +#include "gromacs/utility/fatalerror.h" + +#include "device_information.h" -bool canPerformGpuDetection() +bool canPerformDeviceDetection(std::string* errorMessage) { if (c_binarySupportsGpus && getenv("GMX_DISABLE_GPU_DETECTION") == nullptr) { - return isGpuDetectionFunctional(nullptr); + return isDeviceDetectionFunctional(errorMessage); } else { @@ -61,29 +65,61 @@ bool canPerformGpuDetection() } } -std::vector getCompatibleGpus(const gmx_gpu_info_t& gpu_info) +bool canComputeOnDevice() +{ + bool canComputeOnDevice = false; + if (canPerformDeviceDetection(nullptr)) + { + std::vector> devInfos = findDevices(); + canComputeOnDevice = !getCompatibleDevices(devInfos).empty(); + } + return canComputeOnDevice; +} + +std::vector> +getCompatibleDevices(const std::vector>& deviceInfoList) { // Possible minor over-allocation here, but not important for anything - std::vector compatibleGpus; - compatibleGpus.reserve(gpu_info.n_dev); - for (int i = 0; i < gpu_info.n_dev; i++) + std::vector> compatibleDeviceInfoList; + compatibleDeviceInfoList.reserve(deviceInfoList.size()); + for (const auto& deviceInfo : deviceInfoList) { - assert(gpu_info.deviceInfo); - if (gpu_info_get_stat(gpu_info, i) == DeviceStatus::Compatible) + if (deviceInfo->status == DeviceStatus::Compatible) { - compatibleGpus.push_back(i); + compatibleDeviceInfoList.emplace_back(*deviceInfo); } } - return compatibleGpus; + return compatibleDeviceInfoList; } -const char* getGpuCompatibilityDescription(const gmx_gpu_info_t& gpu_info, int index) +std::string getDeviceCompatibilityDescription(const std::vector>& deviceInfoList, + int deviceId) { - return (index >= gpu_info.n_dev ? c_deviceStateString[DeviceStatus::Nonexistent] - : c_deviceStateString[gpu_info_get_stat(gpu_info, index)]); + return (deviceId >= static_cast(deviceInfoList.size()) + ? c_deviceStateString[DeviceStatus::Nonexistent] + : c_deviceStateString[deviceInfoList[deviceId]->status]); } -void free_gpu_info(const gmx_gpu_info_t* gpu_info) +void serializeDeviceInformations(const std::vector>& deviceInfoList, + gmx::ISerializer* serializer) { - sfree(static_cast(gpu_info->deviceInfo)); // circumvent is_pod check in sfree + int numDevices = deviceInfoList.size(); + serializer->doInt(&numDevices); + for (auto& deviceInfo : deviceInfoList) + { + serializer->doOpaque(reinterpret_cast(deviceInfo.get()), sizeof(DeviceInformation)); + } +} + +std::vector> deserializeDeviceInformations(gmx::ISerializer* serializer) +{ + int numDevices = 0; + serializer->doInt(&numDevices); + std::vector> deviceInfoList(numDevices); + for (int i = 0; i < numDevices; i++) + { + deviceInfoList[i] = std::make_unique(); + serializer->doOpaque(reinterpret_cast(deviceInfoList[i].get()), sizeof(DeviceInformation)); + } + return deviceInfoList; } diff --git a/src/gromacs/hardware/device_management_ocl.cpp b/src/gromacs/hardware/device_management_ocl.cpp index 3cf2eec706..4404efed3c 100644 --- a/src/gromacs/hardware/device_management_ocl.cpp +++ b/src/gromacs/hardware/device_management_ocl.cpp @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2012,2013,2014,2015,2016 by the GROMACS development team. + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team. * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the @@ -34,50 +34,62 @@ * the research papers on the package. Check out http://www.gromacs.org. */ /*! \internal \file - * \brief Define functions for detection and initialization for OpenCL devices. + * \brief Defines the OpenCL implementations of the device management. * * \author Anca Hamuraru * \author Dimitrios Karkoulis * \author Teemu Virolainen * \author Mark Abraham * \author Szilárd Páll + * \author Artem Zhmurov + * + * \ingroup module_hardware */ - #include "gmxpre.h" #include "config.h" -#include -#include -#include -#include - -#include -#ifdef __APPLE__ -# include -#endif - -#include - -#include "gromacs/gpu_utils/ocl_compiler.h" #include "gromacs/gpu_utils/oclraii.h" #include "gromacs/gpu_utils/oclutils.h" -#include "gromacs/hardware/device_information.h" #include "gromacs/hardware/device_management.h" -#include "gromacs/hardware/hw_info.h" -#include "gromacs/utility/cstringutil.h" -#include "gromacs/utility/exceptions.h" #include "gromacs/utility/fatalerror.h" #include "gromacs/utility/smalloc.h" #include "gromacs/utility/stringutil.h" +#include "device_information.h" + +namespace gmx +{ + +/*! \brief Returns an DeviceVendor value corresponding to the input OpenCL vendor name. + * + * \returns DeviceVendor value for the input vendor name + */ +static DeviceVendor getDeviceVendor(const char* vendorName) +{ + if (vendorName) + { + if (strstr(vendorName, "NVIDIA")) + { + return DeviceVendor::Nvidia; + } + else if (strstr(vendorName, "AMD") || strstr(vendorName, "Advanced Micro Devices")) + { + return DeviceVendor::Amd; + } + else if (strstr(vendorName, "Intel")) + { + return DeviceVendor::Intel; + } + } + return DeviceVendor::Unknown; +} + /*! \brief Return true if executing on compatible OS for AMD OpenCL. * * This is assumed to be true for OS X version of at least 10.10.4 and * all other OS flavors. * - * Uses the BSD sysctl() interfaces to extract the kernel version. - * * \return true if version is 14.4 or later (= OS X version 10.10.4), * or OS is not Darwin. */ @@ -89,9 +101,6 @@ static bool runningOnCompatibleOSForAmd() size_t len = sizeof(kernelVersion); mib[0] = CTL_KERN; - mib[1] = KERN_OSRELEASE; - - sysctl(mib, sizeof(mib) / sizeof(mib[0]), kernelVersion, &len, NULL, 0); int major = strtod(kernelVersion, NULL); int minor = strtod(strchr(kernelVersion, '.') + 1, NULL); @@ -103,29 +112,79 @@ static bool runningOnCompatibleOSForAmd() #endif } -namespace gmx +/*! + * \brief Checks that device \c deviceInfo is compatible with GROMACS. + * + * Vendor and OpenCL version support checks are executed an the result + * of these returned. + * + * \param[in] deviceInfo The device info pointer. + * \returns The result of the compatibility checks. + */ +static DeviceStatus isDeviceSupported(const DeviceInformation& deviceInfo) { + if (getenv("GMX_OCL_DISABLE_COMPATIBILITY_CHECK") != nullptr) + { + // Assume the device is compatible because checking has been disabled. + return DeviceStatus::Compatible; + } + + // OpenCL device version check, ensure >= REQUIRED_OPENCL_MIN_VERSION + constexpr unsigned int minVersionMajor = REQUIRED_OPENCL_MIN_VERSION_MAJOR; + constexpr unsigned int minVersionMinor = REQUIRED_OPENCL_MIN_VERSION_MINOR; + + // Based on the OpenCL spec we're checking the version supported by + // the device which has the following format: + // OpenCL + unsigned int deviceVersionMinor, deviceVersionMajor; + const int valuesScanned = std::sscanf(deviceInfo.device_version, "OpenCL %u.%u", + &deviceVersionMajor, &deviceVersionMinor); + const bool versionLargeEnough = + ((valuesScanned == 2) + && ((deviceVersionMajor > minVersionMajor) + || (deviceVersionMajor == minVersionMajor && deviceVersionMinor >= minVersionMinor))); + if (!versionLargeEnough) + { + return DeviceStatus::Incompatible; + } + + /* Only AMD, Intel, and NVIDIA GPUs are supported for now */ + switch (deviceInfo.deviceVendor) + { + case DeviceVendor::Nvidia: return DeviceStatus::Compatible; + case DeviceVendor::Amd: + return runningOnCompatibleOSForAmd() ? DeviceStatus::Compatible : DeviceStatus::Incompatible; + case DeviceVendor::Intel: + return GMX_OPENCL_NB_CLUSTER_SIZE == 4 ? DeviceStatus::Compatible + : DeviceStatus::IncompatibleClusterSize; + default: return DeviceStatus::Incompatible; + } +} /*! \brief Make an error string following an OpenCL API call. * * It is meant to be called with \p status != CL_SUCCESS, but it will * work correctly even if it is called with no OpenCL failure. * + * \todo Make use of this function more. + * * \param[in] message Supplies context, e.g. the name of the API call that returned the error. * \param[in] status OpenCL API status code * \returns A string describing the OpenCL error. */ -static std::string makeOpenClInternalErrorString(const char* message, cl_int status) +inline std::string makeOpenClInternalErrorString(const char* message, cl_int status) { if (message != nullptr) { - return formatString("%s did %ssucceed %d: %s", message, ((status != CL_SUCCESS) ? "not " : ""), - status, ocl_get_error_string(status).c_str()); + return gmx::formatString("%s did %ssucceed %d: %s", message, + ((status != CL_SUCCESS) ? "not " : ""), status, + ocl_get_error_string(status).c_str()); } else { - return formatString("%sOpenCL error encountered %d: %s", ((status != CL_SUCCESS) ? "" : "No "), - status, ocl_get_error_string(status).c_str()); + return gmx::formatString("%sOpenCL error encountered %d: %s", + ((status != CL_SUCCESS) ? "" : "No "), status, + ocl_get_error_string(status).c_str()); } } @@ -141,15 +200,15 @@ static std::string makeOpenClInternalErrorString(const char* message, cl_int sta * \throws std::bad_alloc When out of memory. * \returns Whether the device passed sanity checks */ -static bool isDeviceFunctional(const DeviceInformation* deviceInfo, std::string* errorMessage) +static bool isDeviceFunctional(const DeviceInformation& deviceInfo, std::string* errorMessage) { cl_context_properties properties[] = { - CL_CONTEXT_PLATFORM, reinterpret_cast(deviceInfo->oclPlatformId), 0 + CL_CONTEXT_PLATFORM, reinterpret_cast(deviceInfo.oclPlatformId), 0 }; // uncrustify spacing cl_int status; - auto deviceId = deviceInfo->oclDeviceId; + auto deviceId = deviceInfo.oclDeviceId; ClContext context(clCreateContext(properties, 1, &deviceId, nullptr, nullptr, &status)); if (status != CL_SUCCESS) { @@ -198,56 +257,6 @@ static bool isDeviceFunctional(const DeviceInformation* deviceInfo, std::string* return true; } -/*! - * \brief Checks that device \c deviceInfo is compatible with GROMACS. - * - * Vendor and OpenCL version support checks are executed an the result - * of these returned. - * - * \param[in] deviceInfo The device info pointer. - * \returns The result of the compatibility checks. - */ -static DeviceStatus isDeviceSupported(const DeviceInformation* deviceInfo) -{ - if (getenv("GMX_OCL_DISABLE_COMPATIBILITY_CHECK") != nullptr) - { - // Assume the device is compatible because checking has been disabled. - return DeviceStatus::Compatible; - } - - // OpenCL device version check, ensure >= REQUIRED_OPENCL_MIN_VERSION - constexpr unsigned int minVersionMajor = REQUIRED_OPENCL_MIN_VERSION_MAJOR; - constexpr unsigned int minVersionMinor = REQUIRED_OPENCL_MIN_VERSION_MINOR; - - // Based on the OpenCL spec we're checking the version supported by - // the device which has the following format: - // OpenCL - unsigned int deviceVersionMinor, deviceVersionMajor; - const int valuesScanned = std::sscanf(deviceInfo->device_version, "OpenCL %u.%u", - &deviceVersionMajor, &deviceVersionMinor); - const bool versionLargeEnough = - ((valuesScanned == 2) - && ((deviceVersionMajor > minVersionMajor) - || (deviceVersionMajor == minVersionMajor && deviceVersionMinor >= minVersionMinor))); - if (!versionLargeEnough) - { - return DeviceStatus::Incompatible; - } - - /* Only AMD, Intel, and NVIDIA GPUs are supported for now */ - switch (deviceInfo->deviceVendor) - { - case DeviceVendor::Nvidia: return DeviceStatus::Compatible; - case DeviceVendor::Amd: - return runningOnCompatibleOSForAmd() ? DeviceStatus::Compatible : DeviceStatus::Incompatible; - case DeviceVendor::Intel: - return GMX_OPENCL_NB_CLUSTER_SIZE == 4 ? DeviceStatus::Compatible - : DeviceStatus::IncompatibleClusterSize; - default: return DeviceStatus::Incompatible; - } -} - - /*! \brief Check whether the \c ocl_gpu_device is suitable for use by mdrun * * Runs sanity checks: checking that the runtime can compile a dummy kernel @@ -260,7 +269,7 @@ static DeviceStatus isDeviceSupported(const DeviceInformation* deviceInfo) * \returns A DeviceStatus to indicate if the GPU device is supported and if it was able to run * basic functionality checks. */ -static DeviceStatus checkGpu(size_t deviceId, const DeviceInformation* deviceInfo) +static DeviceStatus checkGpu(size_t deviceId, const DeviceInformation& deviceInfo) { DeviceStatus supportStatus = isDeviceSupported(deviceInfo); @@ -281,32 +290,7 @@ static DeviceStatus checkGpu(size_t deviceId, const DeviceInformation* deviceInf } // namespace gmx -/*! \brief Returns an DeviceVendor value corresponding to the input OpenCL vendor name. - * - * \param[in] vendorName String with OpenCL vendor name. - * \returns DeviceVendor value for the input vendor name - */ -static DeviceVendor getDeviceVendor(const char* vendorName) -{ - if (vendorName) - { - if (strstr(vendorName, "NVIDIA")) - { - return DeviceVendor::Nvidia; - } - else if (strstr(vendorName, "AMD") || strstr(vendorName, "Advanced Micro Devices")) - { - return DeviceVendor::Amd; - } - else if (strstr(vendorName, "Intel")) - { - return DeviceVendor::Intel; - } - } - return DeviceVendor::Unknown; -} - -bool isGpuDetectionFunctional(std::string* errorMessage) +bool isDeviceDetectionFunctional(std::string* errorMessage) { cl_uint numPlatforms; cl_int status = clGetPlatformIDs(0, nullptr, &numPlatforms); @@ -335,7 +319,7 @@ bool isGpuDetectionFunctional(std::string* errorMessage) return foundPlatform; } -void findGpus(gmx_gpu_info_t* gpu_info) +std::vector> findDevices() { cl_uint ocl_platform_count; cl_platform_id* ocl_platform_ids; @@ -348,6 +332,9 @@ void findGpus(gmx_gpu_info_t* gpu_info) req_dev_type = CL_DEVICE_TYPE_CPU; } + int numDevices = 0; + std::vector> deviceInfoList(0); + while (true) { cl_int status = clGetPlatformIDs(0, nullptr, &ocl_platform_count); @@ -386,22 +373,22 @@ void findGpus(gmx_gpu_info_t* gpu_info) if (1 <= ocl_device_count) { - gpu_info->n_dev += ocl_device_count; + numDevices += ocl_device_count; } } - if (1 > gpu_info->n_dev) + if (1 > numDevices) { break; } - snew(gpu_info->deviceInfo, gpu_info->n_dev); + deviceInfoList.resize(numDevices); { int device_index; cl_device_id* ocl_device_ids; - snew(ocl_device_ids, gpu_info->n_dev); + snew(ocl_device_ids, numDevices); device_index = 0; for (unsigned int i = 0; i < ocl_platform_count; i++) @@ -410,8 +397,8 @@ void findGpus(gmx_gpu_info_t* gpu_info) /* If requesting req_dev_type devices fails, just go to the next platform */ if (CL_SUCCESS - != clGetDeviceIDs(ocl_platform_ids[i], req_dev_type, gpu_info->n_dev, - ocl_device_ids, &ocl_device_count)) + != clGetDeviceIDs(ocl_platform_ids[i], req_dev_type, numDevices, ocl_device_ids, + &ocl_device_count)) { continue; } @@ -423,87 +410,86 @@ void findGpus(gmx_gpu_info_t* gpu_info) for (unsigned int j = 0; j < ocl_device_count; j++) { - gpu_info->deviceInfo[device_index].oclPlatformId = ocl_platform_ids[i]; - gpu_info->deviceInfo[device_index].oclDeviceId = ocl_device_ids[j]; + deviceInfoList[device_index] = std::make_unique(); - gpu_info->deviceInfo[device_index].device_name[0] = 0; + deviceInfoList[device_index]->id = device_index; + + deviceInfoList[device_index]->oclPlatformId = ocl_platform_ids[i]; + deviceInfoList[device_index]->oclDeviceId = ocl_device_ids[j]; + + deviceInfoList[device_index]->device_name[0] = 0; clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_NAME, - sizeof(gpu_info->deviceInfo[device_index].device_name), - gpu_info->deviceInfo[device_index].device_name, nullptr); + sizeof(deviceInfoList[device_index]->device_name), + deviceInfoList[device_index]->device_name, nullptr); - gpu_info->deviceInfo[device_index].device_version[0] = 0; + deviceInfoList[device_index]->device_version[0] = 0; clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_VERSION, - sizeof(gpu_info->deviceInfo[device_index].device_version), - gpu_info->deviceInfo[device_index].device_version, nullptr); + sizeof(deviceInfoList[device_index]->device_version), + deviceInfoList[device_index]->device_version, nullptr); - gpu_info->deviceInfo[device_index].vendorName[0] = 0; + deviceInfoList[device_index]->vendorName[0] = 0; clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_VENDOR, - sizeof(gpu_info->deviceInfo[device_index].vendorName), - gpu_info->deviceInfo[device_index].vendorName, nullptr); + sizeof(deviceInfoList[device_index]->vendorName), + deviceInfoList[device_index]->vendorName, nullptr); - gpu_info->deviceInfo[device_index].compute_units = 0; + deviceInfoList[device_index]->compute_units = 0; clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_COMPUTE_UNITS, - sizeof(gpu_info->deviceInfo[device_index].compute_units), - &(gpu_info->deviceInfo[device_index].compute_units), nullptr); + sizeof(deviceInfoList[device_index]->compute_units), + &(deviceInfoList[device_index]->compute_units), nullptr); - gpu_info->deviceInfo[device_index].adress_bits = 0; + deviceInfoList[device_index]->adress_bits = 0; clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_ADDRESS_BITS, - sizeof(gpu_info->deviceInfo[device_index].adress_bits), - &(gpu_info->deviceInfo[device_index].adress_bits), nullptr); + sizeof(deviceInfoList[device_index]->adress_bits), + &(deviceInfoList[device_index]->adress_bits), nullptr); - gpu_info->deviceInfo[device_index].deviceVendor = - getDeviceVendor(gpu_info->deviceInfo[device_index].vendorName); + deviceInfoList[device_index]->deviceVendor = + gmx::getDeviceVendor(deviceInfoList[device_index]->vendorName); clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_WORK_ITEM_SIZES, 3 * sizeof(size_t), - &gpu_info->deviceInfo[device_index].maxWorkItemSizes, nullptr); + &deviceInfoList[device_index]->maxWorkItemSizes, nullptr); clGetDeviceInfo(ocl_device_ids[j], CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), - &gpu_info->deviceInfo[device_index].maxWorkGroupSize, nullptr); - - gpu_info->deviceInfo[device_index].stat = - gmx::checkGpu(device_index, gpu_info->deviceInfo + device_index); + &deviceInfoList[device_index]->maxWorkGroupSize, nullptr); - if (DeviceStatus::Compatible == gpu_info->deviceInfo[device_index].stat) - { - gpu_info->n_dev_compatible++; - } + deviceInfoList[device_index]->status = + gmx::checkGpu(device_index, *deviceInfoList[device_index]); device_index++; } } - gpu_info->n_dev = device_index; + numDevices = device_index; /* Dummy sort of devices - AMD first, then NVIDIA, then Intel */ // TODO: Sort devices based on performance. - if (0 < gpu_info->n_dev) + if (0 < numDevices) { int last = -1; - for (int i = 0; i < gpu_info->n_dev; i++) + for (int i = 0; i < numDevices; i++) { - if (gpu_info->deviceInfo[i].deviceVendor == DeviceVendor::Amd) + if (deviceInfoList[i]->deviceVendor == DeviceVendor::Amd) { last++; if (last < i) { - std::swap(gpu_info->deviceInfo[i], gpu_info->deviceInfo[last]); + std::swap(deviceInfoList[i], deviceInfoList[last]); } } } /* if more than 1 device left to be sorted */ - if ((gpu_info->n_dev - 1 - last) > 1) + if ((numDevices - 1 - last) > 1) { - for (int i = 0; i < gpu_info->n_dev; i++) + for (int i = 0; i < numDevices; i++) { - if (gpu_info->deviceInfo[i].deviceVendor == DeviceVendor::Nvidia) + if (deviceInfoList[i]->deviceVendor == DeviceVendor::Nvidia) { last++; if (last < i) { - std::swap(gpu_info->deviceInfo[i], gpu_info->deviceInfo[last]); + std::swap(deviceInfoList[i], deviceInfoList[last]); } } } @@ -517,18 +503,17 @@ void findGpus(gmx_gpu_info_t* gpu_info) } sfree(ocl_platform_ids); + return deviceInfoList; } -void init_gpu(const DeviceInformation* deviceInfo) +void setActiveDevice(const DeviceInformation& deviceInfo) { - assert(deviceInfo); - // If the device is NVIDIA, for safety reasons we disable the JIT // caching as this is known to be broken at least until driver 364.19; // the cache does not always get regenerated when the source code changes, // e.g. if the path to the kernel sources remains the same - if (deviceInfo->deviceVendor == DeviceVendor::Nvidia) + if (deviceInfo.deviceVendor == DeviceVendor::Nvidia) { // Ignore return values, failing to set the variable does not mean // that something will go wrong later. @@ -541,48 +526,22 @@ void init_gpu(const DeviceInformation* deviceInfo) } } -void free_gpu(const DeviceInformation* /* deviceInfo */) {} - -DeviceInformation* getDeviceInfo(const gmx_gpu_info_t& gpu_info, int deviceId) -{ - if (deviceId < 0 || deviceId >= gpu_info.n_dev) - { - gmx_incons("Invalid GPU deviceId requested"); - } - return &gpu_info.deviceInfo[deviceId]; -} +void releaseDevice(DeviceInformation* /* deviceInfo */) {} -void get_gpu_device_info_string(char* s, const gmx_gpu_info_t& gpu_info, int index) +std::string getDeviceInformationString(const DeviceInformation& deviceInfo) { - assert(s); + bool gpuExists = (deviceInfo.status != DeviceStatus::Nonexistent + && deviceInfo.status != DeviceStatus::NonFunctional); - if (index < 0 && index >= gpu_info.n_dev) + if (!gpuExists) { - return; - } - - DeviceInformation* dinfo = &gpu_info.deviceInfo[index]; - - bool bGpuExists = - (dinfo->stat != DeviceStatus::Nonexistent && dinfo->stat != DeviceStatus::NonFunctional); - - if (!bGpuExists) - { - sprintf(s, "#%d: %s, stat: %s", index, "N/A", c_deviceStateString[dinfo->stat]); + return gmx::formatString("#%d: %s, status: %s", deviceInfo.id, "N/A", + c_deviceStateString[deviceInfo.status]); } else { - sprintf(s, "#%d: name: %s, vendor: %s, device version: %s, stat: %s", index, dinfo->device_name, - dinfo->vendorName, dinfo->device_version, c_deviceStateString[dinfo->stat]); + return gmx::formatString("#%d: name: %s, vendor: %s, device version: %s, status: %s", + deviceInfo.id, deviceInfo.device_name, deviceInfo.vendorName, + deviceInfo.device_version, c_deviceStateString[deviceInfo.status]); } } - -size_t sizeof_gpu_dev_info() -{ - return sizeof(DeviceInformation); -} - -DeviceStatus gpu_info_get_stat(const gmx_gpu_info_t& info, int index) -{ - return info.deviceInfo[index].stat; -} diff --git a/src/gromacs/hardware/gpu_hw_info.h b/src/gromacs/hardware/gpu_hw_info.h deleted file mode 100644 index ff114d1248..0000000000 --- a/src/gromacs/hardware/gpu_hw_info.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * This file is part of the GROMACS molecular simulation package. - * - * Copyright (c) 2012,2013,2014,2015,2017 The GROMACS development team. - * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by - * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, - * and including many others, as listed in the AUTHORS file in the - * top-level source directory and at http://www.gromacs.org. - * - * GROMACS is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * GROMACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with GROMACS; if not, see - * http://www.gnu.org/licenses, or write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * If you want to redistribute modifications to GROMACS, please - * consider that scientific software is very special. Version - * control is crucial - bugs must be traceable. We will be happy to - * consider code for inclusion in the official distribution, but - * derived work must not be called official GROMACS. Details are found - * in the README & COPYING files - if they are missing, get the - * official version at http://www.gromacs.org. - * - * To help us fund GROMACS development, we humbly ask that you cite - * the research papers on the package. Check out http://www.gromacs.org. - */ -#ifndef GMX_HARDWARE_GPU_HW_INFO_H -#define GMX_HARDWARE_GPU_HW_INFO_H - -#include "gromacs/utility/basedefinitions.h" -#include "gromacs/utility/enumerationhelpers.h" - -struct DeviceInformation; - -/*! \brief Information about GPU devices on this physical node. - * - * Includes either CUDA or OpenCL devices. The gmx_hardware_detect - * module initializes it. - * - * \todo Use a std::vector */ -struct gmx_gpu_info_t -{ - //! Did we attempt GPU detection? - gmx_bool bDetectGPUs; - //! Total number of GPU devices detected on this physical node - int n_dev; - //! Information about each GPU device detected on this physical node - DeviceInformation* deviceInfo; - //! Number of GPU devices detected on this physical node that are compatible. - int n_dev_compatible; -}; - -#endif diff --git a/src/gromacs/hardware/hw_info.h b/src/gromacs/hardware/hw_info.h index d09fcfe7d2..b7b796f8b6 100644 --- a/src/gromacs/hardware/hw_info.h +++ b/src/gromacs/hardware/hw_info.h @@ -40,7 +40,7 @@ #include #include -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/utility/basedefinitions.h" namespace gmx @@ -48,6 +48,7 @@ namespace gmx class CpuInfo; class HardwareTopology; } // namespace gmx +struct DeviceInformation; /* Hardware information structure with CPU and GPU information. * It is initialized by gmx_detect_hardware(). @@ -61,8 +62,6 @@ struct gmx_hw_info_t ~gmx_hw_info_t(); /* Data for our local physical node */ - //! Information about GPUs detected on this physical node - gmx_gpu_info_t gpu_info; /*! \brief Number of hardware threads available. * @@ -73,6 +72,7 @@ struct gmx_hw_info_t std::unique_ptr cpuInfo; /* Information about CPU capabilities */ std::unique_ptr hardwareTopology; /* Information about hardware topology */ + std::vector> deviceInfoList; /* Information about GPUs detected on this physical node */ /* Data reduced through MPI over all physical nodes */ diff --git a/src/gromacs/hardware/printhardware.cpp b/src/gromacs/hardware/printhardware.cpp index b7af58092b..bf356ecfa4 100644 --- a/src/gromacs/hardware/printhardware.cpp +++ b/src/gromacs/hardware/printhardware.cpp @@ -66,14 +66,12 @@ static constexpr bool bGPUBinary = (GMX_GPU != 0); /*! \internal \brief * Returns the GPU information text, one GPU per line. */ -static std::string sprint_gpus(const gmx_gpu_info_t& gpu_info) +static std::string sprint_gpus(const std::vector>& deviceInfoList) { - char stmp[STRLEN]; - std::vector gpuStrings; - for (int i = 0; i < gpu_info.n_dev; i++) + std::vector gpuStrings(0); + for (const auto& deviceInfo : deviceInfoList) { - get_gpu_device_info_string(stmp, gpu_info, i); - gpuStrings.push_back(gmx::formatString(" %s", stmp)); + gpuStrings.emplace_back(" " + getDeviceInformationString(*deviceInfo)); } return gmx::joinStrings(gpuStrings, "\n"); } @@ -145,7 +143,7 @@ static std::string detected_hardware_string(const gmx_hw_info_t* hwinfo, bool bF s += gmx::formatString(" %d cores,", hwinfo->ncore_tot); } s += gmx::formatString(" %d logical cores", hwinfo->nhwthread_tot); - if (hwinfo->gpu_info.bDetectGPUs) + if (canPerformDeviceDetection(nullptr)) { s += gmx::formatString(", %d compatible GPU%s", hwinfo->ngpu_compatible_tot, hwinfo->ngpu_compatible_tot == 1 ? "" : "s"); @@ -344,11 +342,12 @@ static std::string detected_hardware_string(const gmx_hw_info_t* hwinfo, bool bF } } - if (bGPUBinary && hwinfo->gpu_info.n_dev > 0) + if (bGPUBinary && !hwinfo->deviceInfoList.empty()) { s += gmx::formatString(" GPU info:\n"); - s += gmx::formatString(" Number of GPUs detected: %d\n", hwinfo->gpu_info.n_dev); - s += sprint_gpus(hwinfo->gpu_info) + "\n"; + s += gmx::formatString(" Number of GPUs detected: %d\n", + static_cast(hwinfo->deviceInfoList.size())); + s += sprint_gpus(hwinfo->deviceInfoList) + "\n"; } return s; } diff --git a/src/gromacs/hardware/tests/CMakeLists.txt b/src/gromacs/hardware/tests/CMakeLists.txt index 9d28379c50..8479ac5399 100644 --- a/src/gromacs/hardware/tests/CMakeLists.txt +++ b/src/gromacs/hardware/tests/CMakeLists.txt @@ -35,5 +35,6 @@ gmx_add_unit_test(HardwareUnitTests hardware-test CPP_SOURCE_FILES cpuinfo.cpp + device_management.cpp hardwaretopology.cpp ) diff --git a/src/gromacs/hardware/tests/device_management.cpp b/src/gromacs/hardware/tests/device_management.cpp new file mode 100644 index 0000000000..cfbcd134fb --- /dev/null +++ b/src/gromacs/hardware/tests/device_management.cpp @@ -0,0 +1,102 @@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 2020, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +/*! \internal \file + * \brief + * Tests for DevicesManager + * + * \author Artem Zhmurov + * \ingroup module_hardware + */ +#include "gmxpre.h" + +#include "gromacs/hardware/device_management.h" + +#include "config.h" + +#include + +#include + +#include "gromacs/hardware/device_information.h" +#include "gromacs/utility/inmemoryserializer.h" +#include "gromacs/utility/stringutil.h" + +namespace +{ + +TEST(DevicesManagerTest, Serialization) +{ + if (canPerformDeviceDetection(nullptr)) + { + std::vector> deviceInfoListIn = findDevices(); + gmx::InMemorySerializer writer; + serializeDeviceInformations(deviceInfoListIn, &writer); + auto buffer = writer.finishAndGetBuffer(); + + gmx::InMemoryDeserializer reader(buffer, false); + std::vector> deviceInfoListOut = + deserializeDeviceInformations(&reader); + + EXPECT_EQ(deviceInfoListOut.size(), deviceInfoListIn.size()) + << "Number of accessible devices changed after serialization/deserialization."; + + for (int deviceId = 0; deviceId < static_cast(deviceInfoListIn.size()); deviceId++) + { + EXPECT_FALSE(deviceInfoListIn[deviceId] == nullptr) << gmx::formatString( + "Device #%d information is nullptr before serialization.", deviceId); + EXPECT_FALSE(deviceInfoListOut[deviceId] == nullptr) << gmx::formatString( + "Device #%d information is nullptr after serialization.", deviceId); + + const DeviceInformation& deviceInfoIn = *deviceInfoListIn[deviceId]; + const DeviceInformation& deviceInfoOut = *deviceInfoListOut[deviceId]; + EXPECT_EQ(deviceInfoIn.status, deviceInfoOut.status) << gmx::formatString( + "Device status changed after serialization/deserialization for device #%d.", deviceId); + + EXPECT_EQ(deviceInfoIn.id, deviceInfoOut.id) << gmx::formatString( + "Device id changed after serialization/deserialization for device #%d.", deviceId); + +#if GMX_GPU_OPENCL + EXPECT_EQ(deviceInfoIn.oclPlatformId, deviceInfoOut.oclPlatformId) << gmx::formatString( + "Device OpenCL platform ID changed after serialization/deserialization for " + "device " + "#%d.", + deviceId); + +#endif // GMX_GPU_OPENCL + } + } +} + +} // namespace diff --git a/src/gromacs/mdlib/forcerec.h b/src/gromacs/mdlib/forcerec.h index 4fbbee3ab2..aed645f186 100644 --- a/src/gromacs/mdlib/forcerec.h +++ b/src/gromacs/mdlib/forcerec.h @@ -47,7 +47,6 @@ struct t_commrec; struct t_forcerec; struct t_filenm; struct t_inputrec; -struct gmx_gpu_info_t; struct gmx_localtop_t; struct gmx_mtop_t; struct gmx_wallcycle; diff --git a/src/gromacs/mdlib/tests/constr.cpp b/src/gromacs/mdlib/tests/constr.cpp index 35d9adce32..ade8dde632 100644 --- a/src/gromacs/mdlib/tests/constr.cpp +++ b/src/gromacs/mdlib/tests/constr.cpp @@ -89,7 +89,7 @@ std::vector getRunnersNames() { runnersNames.emplace_back("SHAKE"); runnersNames.emplace_back("LINCS"); - if (GMX_GPU_CUDA && canComputeOnGpu()) + if (GMX_GPU_CUDA && canComputeOnDevice()) { runnersNames.emplace_back("LINCS_GPU"); } diff --git a/src/gromacs/mdlib/tests/constrtestdata.h b/src/gromacs/mdlib/tests/constrtestdata.h index 65d6e03f56..9adb9c0a61 100644 --- a/src/gromacs/mdlib/tests/constrtestdata.h +++ b/src/gromacs/mdlib/tests/constrtestdata.h @@ -50,7 +50,7 @@ #include #include "gromacs/gmxlib/nrnb.h" -#include "gromacs/gpu_utils/gpu_testutils.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/math/paddedvector.h" #include "gromacs/math/vec.h" #include "gromacs/math/vectypes.h" diff --git a/src/gromacs/mdlib/tests/constrtestrunners.cu b/src/gromacs/mdlib/tests/constrtestrunners.cu index 6b97a80649..62b713cd7b 100644 --- a/src/gromacs/mdlib/tests/constrtestrunners.cu +++ b/src/gromacs/mdlib/tests/constrtestrunners.cu @@ -52,7 +52,7 @@ #include #include "gromacs/gpu_utils/devicebuffer.cuh" -#include "gromacs/hardware/device_management.h" +#include "gromacs/hardware/device_information.h" #include "gromacs/mdlib/lincs_gpu.cuh" #include "gromacs/pbcutil/pbc.h" #include "gromacs/utility/unique_cptr.h" diff --git a/src/gromacs/mdlib/tests/leapfrog.cpp b/src/gromacs/mdlib/tests/leapfrog.cpp index 4ff12126e8..3018d295ca 100644 --- a/src/gromacs/mdlib/tests/leapfrog.cpp +++ b/src/gromacs/mdlib/tests/leapfrog.cpp @@ -63,7 +63,7 @@ #include -#include "gromacs/gpu_utils/gpu_testutils.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/math/vec.h" #include "gromacs/math/vectypes.h" #include "gromacs/mdtypes/mdatom.h" @@ -153,7 +153,7 @@ public: // All runners should be registered here under appropriate conditions // s_runners_["LeapFrogSimple"] = integrateLeapFrogSimple; - if (GMX_GPU_CUDA && canComputeOnGpu()) + if (GMX_GPU_CUDA && canComputeOnDevice()) { s_runners_["LeapFrogGpu"] = integrateLeapFrogGpu; } diff --git a/src/gromacs/mdlib/tests/settle.cpp b/src/gromacs/mdlib/tests/settle.cpp index 41552e2f58..9dc2d9d505 100644 --- a/src/gromacs/mdlib/tests/settle.cpp +++ b/src/gromacs/mdlib/tests/settle.cpp @@ -80,7 +80,7 @@ #include -#include "gromacs/gpu_utils/gpu_testutils.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/math/vec.h" #include "gromacs/math/vectypes.h" #include "gromacs/mdtypes/mdatom.h" @@ -326,7 +326,7 @@ public: //! Store whether any compatible GPUs exist. static bool s_hasCompatibleGpus; //! Before any test is run, work out whether any compatible GPUs exist. - static void SetUpTestCase() { s_hasCompatibleGpus = canComputeOnGpu(); } + static void SetUpTestCase() { s_hasCompatibleGpus = canComputeOnDevice(); } }; bool SettleTest::s_hasCompatibleGpus = false; diff --git a/src/gromacs/mdlib/tests/settletestrunners.cu b/src/gromacs/mdlib/tests/settletestrunners.cu index 6bbf8eb5e3..930f4cb5ba 100644 --- a/src/gromacs/mdlib/tests/settletestrunners.cu +++ b/src/gromacs/mdlib/tests/settletestrunners.cu @@ -52,7 +52,7 @@ #include #include "gromacs/gpu_utils/devicebuffer.cuh" -#include "gromacs/hardware/device_management.h" +#include "gromacs/hardware/device_information.h" #include "gromacs/mdlib/settle_gpu.cuh" #include "gromacs/utility/unique_cptr.h" @@ -82,9 +82,6 @@ void applySettleGpu(SettleTestData* testData, // there is a CUDA-capable device available. GMX_RELEASE_ASSERT(GMX_GPU_CUDA, "CUDA version of SETTLE was called from non-CUDA build."); - // TODO: Here we should check that at least 1 suitable GPU is available - GMX_RELEASE_ASSERT(canPerformGpuDetection(), "Can't detect CUDA-capable GPUs."); - DeviceInformation deviceInfo; const DeviceContext deviceContext(deviceInfo); const DeviceStream deviceStream(deviceContext, DeviceStreamPriority::Normal, false); diff --git a/src/gromacs/mdrun/runner.cpp b/src/gromacs/mdrun/runner.cpp index ef536514d1..f4938304ff 100644 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@ -774,7 +774,7 @@ int Mdrunner::mdrunner() gmx_print_detected_hardware(fplog, isSimulationMasterRank && isMasterSim(ms), mdlog, hwinfo); - std::vector gpuIdsToUse = makeGpuIdsToUse(hwinfo->gpu_info, hw_opt.gpuIdsAvailable); + std::vector gpuIdsToUse = makeGpuIdsToUse(hwinfo->deviceInfoList, hw_opt.gpuIdsAvailable); // Print citation requests after all software/hardware printing pleaseCiteGromacs(fplog); @@ -1729,7 +1729,7 @@ int Mdrunner::mdrunner() } // FIXME: this is only here to manually unpin mdAtoms->chargeA_ and state->x, - // before we destroy the GPU context(s) in free_gpu(). + // before we destroy the GPU context(s) // Pinned buffers are associated with contexts in CUDA. // As soon as we destroy GPU contexts after mdrunner() exits, these lines should go. mdAtoms.reset(nullptr); @@ -1743,18 +1743,17 @@ int Mdrunner::mdrunner() sfree(disresdata); sfree(oriresdata); - if (hwinfo->gpu_info.n_dev > 0) + if (!hwinfo->deviceInfoList.empty()) { /* stop the GPU profiler (only CUDA) */ stopGpuProfiler(); } /* With tMPI we need to wait for all ranks to finish deallocation before - * destroying the CUDA context in free_gpu() as some tMPI ranks may be sharing + * destroying the CUDA context as some tMPI ranks may be sharing * GPU and context. * - * This is not a concern in OpenCL where we use one context per rank which - * is freed in nbnxn_gpu_free(). + * This is not a concern in OpenCL where we use one context per rank. * * Note: it is safe to not call the barrier on the ranks which do not use GPU, * but it is easier and more futureproof to call it on the whole node. @@ -1769,8 +1768,7 @@ int Mdrunner::mdrunner() { physicalNodeComm.barrier(); } - - free_gpu(deviceInfo); + releaseDevice(deviceInfo); /* Does what it says */ print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime()); diff --git a/src/gromacs/mdtypes/state_propagator_data_gpu_impl_gpu.cpp b/src/gromacs/mdtypes/state_propagator_data_gpu_impl_gpu.cpp index 995976b461..e290b73f4a 100644 --- a/src/gromacs/mdtypes/state_propagator_data_gpu_impl_gpu.cpp +++ b/src/gromacs/mdtypes/state_propagator_data_gpu_impl_gpu.cpp @@ -48,7 +48,6 @@ # include "gromacs/gpu_utils/device_stream_manager.h" # include "gromacs/gpu_utils/devicebuffer.h" -# include "gromacs/gpu_utils/gputraits.h" # include "gromacs/math/vectypes.h" # include "gromacs/mdtypes/state_propagator_data_gpu.h" # include "gromacs/timing/wallcycle.h" diff --git a/src/gromacs/nbnxm/cuda/nbnxm_cuda_data_mgmt.cu b/src/gromacs/nbnxm/cuda/nbnxm_cuda_data_mgmt.cu index cc1b6f37ea..ea1261ee2f 100644 --- a/src/gromacs/nbnxm/cuda/nbnxm_cuda_data_mgmt.cu +++ b/src/gromacs/nbnxm/cuda/nbnxm_cuda_data_mgmt.cu @@ -51,11 +51,13 @@ // TODO Remove this comment when the above order issue is resolved #include "gromacs/gpu_utils/cudautils.cuh" +#include "gromacs/gpu_utils/device_context.h" #include "gromacs/gpu_utils/device_stream_manager.h" #include "gromacs/gpu_utils/gpu_utils.h" #include "gromacs/gpu_utils/gpueventsynchronizer.cuh" #include "gromacs/gpu_utils/pmalloc_cuda.h" #include "gromacs/hardware/device_information.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/math/vectypes.h" #include "gromacs/mdlib/force_flags.h" #include "gromacs/mdtypes/interaction_const.h" diff --git a/src/gromacs/nbnxm/gpu_data_mgmt.h b/src/gromacs/nbnxm/gpu_data_mgmt.h index 21fc8174b5..a1ee291ae8 100644 --- a/src/gromacs/nbnxm/gpu_data_mgmt.h +++ b/src/gromacs/nbnxm/gpu_data_mgmt.h @@ -51,7 +51,6 @@ #include "gromacs/mdtypes/locality.h" struct NbnxmGpu; -struct gmx_gpu_info_t; struct DeviceInformation; struct gmx_wallclock_gpu_nbnxn_t; struct nbnxn_atomdata_t; diff --git a/src/gromacs/nbnxm/opencl/nbnxm_ocl_data_mgmt.cpp b/src/gromacs/nbnxm/opencl/nbnxm_ocl_data_mgmt.cpp index 58d9624e17..f47d754e2f 100644 --- a/src/gromacs/nbnxm/opencl/nbnxm_ocl_data_mgmt.cpp +++ b/src/gromacs/nbnxm/opencl/nbnxm_ocl_data_mgmt.cpp @@ -55,7 +55,7 @@ #include "gromacs/gpu_utils/device_stream_manager.h" #include "gromacs/gpu_utils/oclutils.h" #include "gromacs/hardware/device_information.h" -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/math/vectypes.h" #include "gromacs/mdlib/force_flags.h" #include "gromacs/mdtypes/interaction_const.h" diff --git a/src/gromacs/taskassignment/taskassignment.cpp b/src/gromacs/taskassignment/taskassignment.cpp index 1688e69293..2a823e5728 100644 --- a/src/gromacs/taskassignment/taskassignment.cpp +++ b/src/gromacs/taskassignment/taskassignment.cpp @@ -330,7 +330,7 @@ GpuTaskAssignments GpuTaskAssignmentsBuilder::build(const std::vector& gpuI userGpuTaskAssignment.size(), host, numGpuTasksOnThisNode))); } // Did the user choose compatible GPUs? - checkUserGpuIds(hardwareInfo.gpu_info, gpuIdsToUse, userGpuTaskAssignment); + checkUserGpuIds(hardwareInfo.deviceInfoList, gpuIdsToUse, userGpuTaskAssignment); gpuIdsForTaskAssignment = userGpuTaskAssignment; } @@ -433,8 +433,8 @@ DeviceInformation* GpuTaskAssignments::initDevice(int* deviceId) const if (gpuTaskMapping != gpuTaskAssignment.end()) { *deviceId = gpuTaskMapping->deviceId_; - deviceInfo = getDeviceInfo(hardwareInfo_.gpu_info, *deviceId); - init_gpu(deviceInfo); + deviceInfo = hardwareInfo_.deviceInfoList[*deviceId].get(); + setActiveDevice(*deviceInfo); } return deviceInfo; } diff --git a/src/gromacs/taskassignment/usergpuids.cpp b/src/gromacs/taskassignment/usergpuids.cpp index 275dbfd43b..fff9d0495e 100644 --- a/src/gromacs/taskassignment/usergpuids.cpp +++ b/src/gromacs/taskassignment/usergpuids.cpp @@ -49,6 +49,7 @@ #include #include +#include "gromacs/hardware/device_information.h" #include "gromacs/hardware/device_management.h" #include "gromacs/hardware/hw_info.h" #include "gromacs/utility/exceptions.h" @@ -136,25 +137,30 @@ std::vector parseUserGpuIdString(const std::string& gpuIdString) return digits; } -std::vector makeGpuIdsToUse(const gmx_gpu_info_t& gpuInfo, const std::string& gpuIdsAvailableString) +std::vector makeGpuIdsToUse(const std::vector>& deviceInfoList, + const std::string& gpuIdsAvailableString) { - auto compatibleGpus = getCompatibleGpus(gpuInfo); - std::vector gpuIdsAvailable = parseUserGpuIdString(gpuIdsAvailableString); + auto compatibleDeviceInfoList = getCompatibleDevices(deviceInfoList); + std::vector gpuIdsAvailable = parseUserGpuIdString(gpuIdsAvailableString); + std::vector gpuIdsToUse; if (gpuIdsAvailable.empty()) { - return compatibleGpus; + for (const auto& compatibleDeviceInfo : compatibleDeviceInfoList) + { + gpuIdsToUse.emplace_back(compatibleDeviceInfo.get().id); + } + return gpuIdsToUse; } - std::vector gpuIdsToUse; gpuIdsToUse.reserve(gpuIdsAvailable.size()); std::vector availableGpuIdsThatAreIncompatible; for (const auto& availableGpuId : gpuIdsAvailable) { bool availableGpuIsCompatible = false; - for (const auto& compatibleGpuId : compatibleGpus) + for (const auto& compatibleDeviceInfo : compatibleDeviceInfoList) { - if (availableGpuId == compatibleGpuId) + if (availableGpuId == compatibleDeviceInfo.get().id) { availableGpuIsCompatible = true; break; @@ -217,9 +223,9 @@ std::string makeGpuIdString(const std::vector& gpuIds, int totalNumberOfTas return formatAndJoin(resultGpuIds, ",", StringFormatter("%d")); } -void checkUserGpuIds(const gmx_gpu_info_t& gpu_info, - const std::vector& compatibleGpus, - const std::vector& gpuIds) +void checkUserGpuIds(const std::vector>& deviceInfoList, + const std::vector& compatibleGpus, + const std::vector& gpuIds) { bool foundIncompatibleGpuIds = false; std::string message = @@ -231,7 +237,7 @@ void checkUserGpuIds(const gmx_gpu_info_t& gpu_info, { foundIncompatibleGpuIds = true; message += gmx::formatString(" GPU #%d: %s\n", gpuId, - getGpuCompatibilityDescription(gpu_info, gpuId)); + getDeviceCompatibilityDescription(deviceInfoList, gpuId).c_str()); } } if (foundIncompatibleGpuIds) diff --git a/src/gromacs/taskassignment/usergpuids.h b/src/gromacs/taskassignment/usergpuids.h index 9d9bef4967..e85d15bf18 100644 --- a/src/gromacs/taskassignment/usergpuids.h +++ b/src/gromacs/taskassignment/usergpuids.h @@ -1,7 +1,7 @@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by + * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@ -49,12 +49,13 @@ #include +#include #include #include #include "gromacs/utility/arrayref.h" -struct gmx_gpu_info_t; +struct DeviceInformation; namespace gmx { @@ -83,7 +84,7 @@ std::vector parseUserGpuIdString(const std::string& gpuIdString); * all compatible GPUs on this physical node. Otherwise, check the * user specified compatible GPUs and return their IDs. * - * \param[in] gpuInfo Information detected about GPUs on this physical node + * \param[in] deviceInfoList Information on the GPUs on this physical node. * \param[in] gpuIdsAvailableString String like "013" or "0,1,3" typically * supplied by the user to mdrun -gpu_id. * Must contain only unique decimal digits, or only decimal @@ -99,7 +100,8 @@ std::vector parseUserGpuIdString(const std::string& gpuIdString); * InvalidInputError If gpuIdsAvailableString specifies GPU IDs that are * not compatible. */ -std::vector makeGpuIdsToUse(const gmx_gpu_info_t& gpuInfo, const std::string& gpuIdsAvailableString); +std::vector makeGpuIdsToUse(const std::vector>& deviceInfoList, + const std::string& gpuIdsAvailableString); /*! \brief Parse a GPU ID specifier string into a container describing device ID to task mapping. * @@ -163,16 +165,16 @@ std::string makeGpuIdString(const std::vector& gpuIds, int totalNumberOfTas * infrastructure to do a good job of coordinating error messages and * behaviour across MPMD ranks and multiple simulations. * - * \param[in] gpu_info Information detected about GPUs + * \param[in] deviceInfoList Information on the GPUs on this physical node. * \param[in] compatibleGpus Vector of GPUs that are compatible * \param[in] gpuIds The GPU IDs selected by the user. * * \throws std::bad_alloc If out of memory * InconsistentInputError If the assigned GPUs are not valid */ -void checkUserGpuIds(const gmx_gpu_info_t& gpu_info, - const std::vector& compatibleGpus, - const std::vector& gpuIds); +void checkUserGpuIds(const std::vector>& deviceInfoList, + const std::vector& compatibleGpus, + const std::vector& gpuIds); } // namespace gmx diff --git a/src/programs/mdrun/tests/pmetest.cpp b/src/programs/mdrun/tests/pmetest.cpp index 4b5aadda1b..fbc0e231cf 100644 --- a/src/programs/mdrun/tests/pmetest.cpp +++ b/src/programs/mdrun/tests/pmetest.cpp @@ -55,9 +55,8 @@ #include #include "gromacs/ewald/pme.h" -#include "gromacs/gpu_utils/gpu_testutils.h" #include "gromacs/hardware/detecthardware.h" -#include "gromacs/hardware/gpu_hw_info.h" +#include "gromacs/hardware/device_management.h" #include "gromacs/trajectory/energyframe.h" #include "gromacs/utility/cstringutil.h" #include "gromacs/utility/gmxmpi.h" @@ -98,7 +97,7 @@ bool PmeTest::s_hasCompatibleGpus = false; void PmeTest::SetUpTestCase() { - s_hasCompatibleGpus = canComputeOnGpu(); + s_hasCompatibleGpus = canComputeOnDevice(); } void PmeTest::runTest(const RunModesList& runModes)