Introduce DeviceStreamManager
authorArtem Zhmurov <zhmurov@gmail.com>
Thu, 20 Feb 2020 15:50:29 +0000 (16:50 +0100)
committerMark Abraham <mark.j.abraham@gmail.com>
Mon, 23 Mar 2020 03:46:37 +0000 (04:46 +0100)
Make a separate object that will be handling the creation,
management and destruction of the GPU context and streams.
It is detached from the rest of the code in this patch,
but will be attached in the follow-up.

Refs #3316
Refs #3311

Change-Id: I2c59b930ac266d89fafe9e0172b83f07e9858f0b

src/gromacs/gpu_utils/CMakeLists.txt
src/gromacs/gpu_utils/device_stream.cpp
src/gromacs/gpu_utils/device_stream_manager.cpp [new file with mode: 0644]
src/gromacs/gpu_utils/device_stream_manager.h [new file with mode: 0644]
src/gromacs/gpu_utils/tests/CMakeLists.txt
src/gromacs/gpu_utils/tests/device_stream_manager.cpp [new file with mode: 0644]
src/gromacs/gpu_utils/tests/gputest.cpp
src/gromacs/gpu_utils/tests/gputest.h
src/gromacs/gpu_utils/tests/hostallocator.cpp
src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp

index ce70d0b049adbb5cc8dc099d144ecfc1c5bb0e64..e8c02ccf31e2f28e4937f6a170ef5a2017d108cd 100644 (file)
@@ -38,6 +38,7 @@
 
 gmx_add_libgromacs_sources(
         clfftinitializer.cpp
+        device_stream_manager.cpp
         hostallocator.cpp
         gpu_utils.cpp
         gpu_testutils.cpp
@@ -59,7 +60,10 @@ elseif(GMX_USE_CUDA)
         pinning.cu
         pmalloc_cuda.cu
         )
-elseif()
+    gmx_compile_cpp_as_cuda(
+        device_stream_manager.cpp
+        )
+else()
     gmx_add_libgromacs_sources(
         device_stream.cpp
     )
index c38425caab406f7212a4f746ce4611dec15a1934..7f05de6b321999f905dcf7a3afda9174dc7a3eec 100644 (file)
@@ -59,4 +59,4 @@ bool DeviceStream::isValid() const
     return false;
 }
 
-void DeviceStream::synchronize() const {}
\ No newline at end of file
+void DeviceStream::synchronize() const {}
diff --git a/src/gromacs/gpu_utils/device_stream_manager.cpp b/src/gromacs/gpu_utils/device_stream_manager.cpp
new file mode 100644 (file)
index 0000000..1c8228e
--- /dev/null
@@ -0,0 +1,164 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*! \internal \file
+ *
+ * \brief Implements GPU stream manager.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Artem Zhmurov <zhmurov@gmail.com>
+ *
+ * \ingroup module_gpu_utils
+ */
+#include "gmxpre.h"
+
+#include "device_stream_manager.h"
+
+#include "gromacs/gpu_utils/device_context.h"
+#include "gromacs/gpu_utils/device_stream.h"
+#include "gromacs/gpu_utils/gputraits.h"
+#include "gromacs/utility/enumerationhelpers.h"
+#include "gromacs/utility/exceptions.h"
+#include "gromacs/utility/gmxassert.h"
+#include "gromacs/utility/stringutil.h"
+
+namespace gmx
+{
+
+/*! \libinternal
+ * \brief Impl class to manages the lifetime of the GPU streams.
+ *
+ * If supported by the GPU API, the available runtime and the
+ * indicated device, some streams will be configured at high
+ * priority. Otherwise, all streams will share the default priority
+ * appropriate to the situation.
+ */
+class DeviceStreamManager::Impl
+{
+public:
+    /*! \brief Constructor.
+     *
+     * \throws InternalError  If any of the required resources could not be initialized.
+     */
+    Impl(const DeviceInformation& deviceInfo,
+         bool                     useGpuForPme,
+         bool                     havePpDomainDecomposition,
+         bool                     doGpuPmePpTransfer,
+         bool                     useGpuForUpdate,
+         bool                     useTiming);
+    ~Impl();
+
+    //! Device context.
+    DeviceContext context_;
+    //! GPU command streams.
+    EnumerationArray<DeviceStreamType, DeviceStream> streams_;
+};
+
+// DeviceStreamManager::Impl
+DeviceStreamManager::Impl::Impl(const DeviceInformation& deviceInfo,
+                                const bool               useGpuForPme,
+                                const bool               havePpDomainDecomposition,
+                                const bool               doGpuPmePpTransfer,
+                                const bool               useGpuForUpdate,
+                                const bool               useTiming) :
+    context_(deviceInfo)
+{
+    try
+    {
+        streams_[DeviceStreamType::NonBondedLocal].init(context_, DeviceStreamPriority::Normal, useTiming);
+
+        if (useGpuForPme)
+        {
+            /* Creating a PME GPU stream:
+             * - default high priority with CUDA
+             * - no priorities implemented yet with OpenCL; see #2532
+             */
+            streams_[DeviceStreamType::Pme].init(context_, DeviceStreamPriority::High, useTiming);
+        }
+
+        if (havePpDomainDecomposition)
+        {
+            streams_[DeviceStreamType::NonBondedNonLocal].init(context_, DeviceStreamPriority::High,
+                                                               useTiming);
+        }
+        // Update stream is used both for coordinates transfers and for GPU update/constraints
+        if (useGpuForPme || useGpuForUpdate)
+        {
+            streams_[DeviceStreamType::UpdateAndConstraints].init(
+                    context_, DeviceStreamPriority::Normal, useTiming);
+        }
+        if (doGpuPmePpTransfer)
+        {
+            streams_[DeviceStreamType::PmePpTransfer].init(context_, DeviceStreamPriority::Normal, useTiming);
+        }
+    }
+    GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
+}
+
+DeviceStreamManager::Impl::~Impl() = default;
+
+// DeviceStreamManager
+DeviceStreamManager::DeviceStreamManager(const DeviceInformation& deviceInfo,
+                                         const bool               useGpuForPme,
+                                         const bool               havePpDomainDecomposition,
+                                         const bool               doGpuPmePpTransfer,
+                                         const bool               useGpuForUpdate,
+                                         const bool               useTiming) :
+    impl_(new Impl(deviceInfo, useGpuForPme, havePpDomainDecomposition, doGpuPmePpTransfer, useGpuForUpdate, useTiming))
+{
+}
+
+DeviceStreamManager::~DeviceStreamManager() = default;
+
+const DeviceInformation& DeviceStreamManager::deviceInfo() const
+{
+    return impl_->context_.deviceInfo();
+}
+
+const DeviceContext& DeviceStreamManager::context() const
+{
+    return impl_->context_;
+}
+
+const DeviceStream& DeviceStreamManager::stream(DeviceStreamType streamToGet) const
+{
+    return impl_->streams_[streamToGet];
+}
+
+bool DeviceStreamManager::streamIsValid(DeviceStreamType streamToCheck) const
+{
+    return impl_->streams_[streamToCheck].isValid();
+}
+
+} // namespace gmx
diff --git a/src/gromacs/gpu_utils/device_stream_manager.h b/src/gromacs/gpu_utils/device_stream_manager.h
new file mode 100644 (file)
index 0000000..4cfa616
--- /dev/null
@@ -0,0 +1,142 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*! \libinternal \file
+ *
+ * \brief This file declares a manager of GPU context and streams needed for
+ * running workloads on GPUs.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Artem Zhmurov <zhmurov@gmail.com>
+ *
+ * \inlibraryapi
+ * \ingroup module_gpu_utils
+ */
+#ifndef GMX_GPU_UTILS_GPUSTREAMMANAGER_H
+#define GMX_GPU_UTILS_GPUSTREAMMANAGER_H
+
+#include <string>
+
+#include "gromacs/utility/classhelpers.h"
+
+class DeviceContext;
+struct DeviceInformation;
+class DeviceStream;
+
+namespace gmx
+{
+
+/*! \brief Class enum to describe the different logical streams used
+ * for GPU work.
+ *
+ * Whether the actual streams differ is an implementation detail of
+ * the manager class.
+ */
+enum class DeviceStreamType : int
+{
+    //! Stream primarily for short-ranged local nonbonded work.
+    NonBondedLocal,
+    //! Stream primarily for short-ranged nonlocal nonbonded work.
+    NonBondedNonLocal,
+    //! Stream primarily for PME work.
+    Pme,
+    //! Stream primarily for data exchange between PME and PP ranks.
+    PmePpTransfer,
+    //! Stream primarily for update and constraints.
+    UpdateAndConstraints,
+    //! Conventional termination of the enumeration.
+    Count
+};
+
+/*! \libinternal
+ * \brief Device stream and context manager.
+ *
+ * Manages the lifetime of the GPU streams and their association
+ * with context and device information that is needed to use them.
+ *
+ * If supported by the GPU API, the available runtime and the
+ * indicated device, some streams will be configured at high
+ * priority. Otherwise, all streams will share the default priority
+ * appropriate to the situation.
+ */
+class DeviceStreamManager
+{
+public:
+    /*! \brief Constructor.
+     *
+     * \throws InternalError  If any of the required resources could not be initialized.
+     */
+    DeviceStreamManager(const DeviceInformation& deviceInfo,
+                        bool                     useGpuForPme,
+                        bool                     havePpDomainDecomposition,
+                        bool                     doGpuPmePpTransfer,
+                        bool                     useGpuForUpdate,
+                        bool                     useTiming);
+    ~DeviceStreamManager();
+
+    /*! \brief Get the device information object of the associated device.
+     *
+     * \returns reference to device info.
+     */
+    const DeviceInformation& deviceInfo() const;
+
+    /*! \brief Returns a handle to the GPU context.
+     *
+     * \todo This relies on the fact that only one unique device
+     * is described by nonbondedDeviceInfo and pmeDeviceInfo.
+     */
+    const DeviceContext& context() const;
+
+    /*! \brief Returns a handle to the requested GPU stream.
+     *
+     * \param[in] streamToGet Which stream to get.
+     */
+    const DeviceStream& stream(DeviceStreamType streamToGet) const;
+
+    /*! \brief Return whether the requested GPU stream is valid for use.
+     *
+     * \param[in] streamToCheck Which stream to check.
+     *
+     * \returns Whether the stream was initialized.
+     */
+    bool streamIsValid(DeviceStreamType streamToCheck) const;
+
+private:
+    class Impl;
+    PrivateImplPointer<Impl> impl_;
+};
+
+} // namespace gmx
+
+#endif
index f1f977877913aa643752ac48adf09a3cfa098793..792b481f8a329438e932eb8859f4e0bf3e92e47b 100644 (file)
@@ -51,14 +51,15 @@ gmx_add_unit_test(GpuUtilsUnitTests gpu_utils-test
         typecasts_runner.cu
 
     CUDA_CPP_SOURCE_FILES
+        device_stream_manager.cpp
         pinnedmemorychecker.cpp
         typecasts.cpp
         typecasts_runner.cpp
-        
+
     OPENCL_CPP_SOURCE_FILES
         devicetransfers_ocl.cpp
-    
+
     NON_GPU_CPP_SOURCE_FILES
+        device_stream_manager.cpp
         devicetransfers.cpp
-    
     )
diff --git a/src/gromacs/gpu_utils/tests/device_stream_manager.cpp b/src/gromacs/gpu_utils/tests/device_stream_manager.cpp
new file mode 100644 (file)
index 0000000..1c0330e
--- /dev/null
@@ -0,0 +1,239 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*! \internal \file
+ * \brief Tests GPU stream manager
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \author Artem Zhmurov <zhmurov@gmail.com>
+ *
+ * \ingroup module_gpu_utils
+ */
+#include "gmxpre.h"
+
+#include "gromacs/gpu_utils/device_stream_manager.h"
+
+#include "config.h"
+
+#include <initializer_list>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "gromacs/utility/enumerationhelpers.h"
+
+#include "gputest.h"
+
+namespace gmx
+{
+
+namespace test
+{
+
+namespace
+{
+
+//! GPU device stream names for outputs.
+const EnumerationArray<DeviceStreamType, std::string> c_deviceStreamNames = {
+    { "non-bonded local", "non-bonded non-local", "PME", "PME-PP transfer", "update" }
+};
+
+//! Test fixture
+class DeviceStreamManagerTest : public GpuTest
+{
+public:
+    //! Helper function to implement readable testing
+    void expectValidStreams(DeviceStreamManager* manager, std::initializer_list<DeviceStreamType> types)
+    {
+        if (canExpectValidStreams_)
+        {
+            for (const DeviceStreamType type : types)
+            {
+                SCOPED_TRACE("Testing " + c_deviceStreamNames[type] + " stream.");
+                EXPECT_TRUE(manager->streamIsValid(type));
+            }
+        }
+    }
+    //! Helper function to implement readable testing
+    void expectInvalidStreams(DeviceStreamManager* manager, std::initializer_list<DeviceStreamType> types)
+    {
+        for (const DeviceStreamType type : types)
+        {
+            SCOPED_TRACE("Testing " + c_deviceStreamNames[type] + " stream.");
+            EXPECT_FALSE(manager->streamIsValid(type));
+        }
+    }
+
+    /*! \brief Non-GPU builds return nullptr instead of streams,
+     * so we have to expect that in such build configurations. */
+    const bool canExpectValidStreams_ = (GMX_GPU != GMX_GPU_NONE);
+};
+
+TEST_F(DeviceStreamManagerTest, CorrectStreamsAreReturnedOnNonbondedDevice)
+{
+    // It would be nice to test that the priority is high when it can
+    // be, but that requires calling the same API calls we're testing
+    // that we've called, so it is not very useful.
+    const bool useTiming = false;
+
+    // TODO Is it enough to only test one device?
+    for (const auto* deviceInfo : getDeviceInfos())
+    {
+        EXPECT_FALSE(deviceInfo == nullptr)
+                << "Device information should be provided for the GPU builds.";
+        // Test all the different cases successively.
+
+        {
+            SCOPED_TRACE("No DD, no PME rank, no GPU update");
+            bool                useGpuForPme              = false;
+            bool                havePpDomainDecomposition = false;
+            bool                doGpuPmePpTransfer        = false;
+            bool                useGpuForUpdate           = false;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::NonBondedLocal });
+            expectInvalidStreams(&manager, { DeviceStreamType::NonBondedNonLocal,
+                                             DeviceStreamType::Pme, DeviceStreamType::PmePpTransfer,
+                                             DeviceStreamType::UpdateAndConstraints });
+        }
+
+        {
+            SCOPED_TRACE("With DD, no PME rank, no GPU update");
+            bool                useGpuForPme              = false;
+            bool                havePpDomainDecomposition = true;
+            bool                doGpuPmePpTransfer        = false;
+            bool                useGpuForUpdate           = false;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::NonBondedNonLocal });
+            expectInvalidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::PmePpTransfer,
+                                             DeviceStreamType::UpdateAndConstraints });
+        }
+
+        {
+            SCOPED_TRACE("No DD, with PME rank, no GPU update");
+            bool                useGpuForPme              = true;
+            bool                havePpDomainDecomposition = false;
+            bool                doGpuPmePpTransfer        = true;
+            bool                useGpuForUpdate           = false;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::PmePpTransfer,
+                                           DeviceStreamType::UpdateAndConstraints });
+            expectInvalidStreams(&manager, { DeviceStreamType::NonBondedNonLocal });
+        }
+
+        {
+            SCOPED_TRACE("With DD, with PME rank, no GPU update");
+            bool                useGpuForPme              = true;
+            bool                havePpDomainDecomposition = true;
+            bool                doGpuPmePpTransfer        = true;
+            bool                useGpuForUpdate           = false;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::NonBondedNonLocal, DeviceStreamType::PmePpTransfer,
+                                           DeviceStreamType::UpdateAndConstraints });
+            expectInvalidStreams(&manager, {});
+        }
+
+        {
+            SCOPED_TRACE("No DD, no PME rank, with GPU update");
+            bool                useGpuForPme              = false;
+            bool                havePpDomainDecomposition = false;
+            bool                doGpuPmePpTransfer        = false;
+            bool                useGpuForUpdate           = true;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::UpdateAndConstraints });
+            expectInvalidStreams(&manager, { DeviceStreamType::NonBondedNonLocal,
+                                             DeviceStreamType::Pme, DeviceStreamType::PmePpTransfer });
+        }
+
+        {
+            SCOPED_TRACE("With DD, no PME rank, with GPU update");
+            bool                useGpuForPme              = false;
+            bool                havePpDomainDecomposition = true;
+            bool                doGpuPmePpTransfer        = false;
+            bool                useGpuForUpdate           = true;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::NonBondedLocal, DeviceStreamType::NonBondedNonLocal,
+                                           DeviceStreamType::UpdateAndConstraints });
+            expectInvalidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::PmePpTransfer });
+        }
+
+        {
+            SCOPED_TRACE("No DD, with PME rank, with GPU update");
+            bool                useGpuForPme              = true;
+            bool                havePpDomainDecomposition = false;
+            bool                doGpuPmePpTransfer        = true;
+            bool                useGpuForUpdate           = true;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::PmePpTransfer,
+                                           DeviceStreamType::UpdateAndConstraints });
+            expectInvalidStreams(&manager, { DeviceStreamType::NonBondedNonLocal });
+        }
+
+        {
+            SCOPED_TRACE("With DD, with PME rank, with GPU update");
+            bool                useGpuForPme              = true;
+            bool                havePpDomainDecomposition = true;
+            bool                doGpuPmePpTransfer        = true;
+            bool                useGpuForUpdate           = true;
+            DeviceStreamManager manager(*deviceInfo, useGpuForPme, havePpDomainDecomposition,
+                                        doGpuPmePpTransfer, useGpuForUpdate, useTiming);
+
+            expectValidStreams(&manager, { DeviceStreamType::Pme, DeviceStreamType::NonBondedLocal,
+                                           DeviceStreamType::NonBondedNonLocal, DeviceStreamType::PmePpTransfer,
+                                           DeviceStreamType::UpdateAndConstraints });
+        }
+    }
+}
+
+} // namespace
+} // namespace test
+} // namespace gmx
index 2012d97da3b8bb56c3ba1f0e18154cb1eadf8bb0..a862361586ebf8b7b3d0e85e7630095a242d132c 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -59,6 +59,7 @@ GpuTest::GpuTest()
     if (isGpuDetectionFunctional(nullptr))
     {
         findGpus(gpuInfo_);
+        compatibleGpuIds_ = getCompatibleGpus(*gpuInfo_);
     }
     // Failing to find valid GPUs does not require further action
 }
@@ -69,9 +70,20 @@ GpuTest::~GpuTest()
     sfree(gpuInfo_);
 }
 
-bool GpuTest::haveValidGpus() const
+bool GpuTest::haveCompatibleGpus() const
 {
-    return gpuInfo_->n_dev_compatible > 0;
+    return !compatibleGpuIds_.empty();
+}
+
+std::vector<const DeviceInformation*> GpuTest::getDeviceInfos() const
+{
+    std::vector<const DeviceInformation*> deviceInfos;
+    deviceInfos.reserve(compatibleGpuIds_.size());
+    for (const auto& id : compatibleGpuIds_)
+    {
+        deviceInfos.emplace_back(getDeviceInfo(*gpuInfo_, id));
+    }
+    return deviceInfos;
 }
 
 } // namespace test
index 788725170fec74bc3610fc7917e977129483ede3..a78b00defe99a9bae91804c8e5605403be8f1714 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -45,6 +45,7 @@
 
 #include <gtest/gtest.h>
 
+struct DeviceInformation;
 struct gmx_gpu_info_t;
 
 namespace gmx
@@ -57,11 +58,15 @@ class GpuTest : public ::testing::Test
 public:
     //! Information about GPUs that are present.
     gmx_gpu_info_t* gpuInfo_;
+    //! Contains the IDs of all compatible GPUs
+    std::vector<int> compatibleGpuIds_;
 
     GpuTest();
     ~GpuTest() override;
-    //! Getter for convenience in testing
-    bool haveValidGpus() const;
+    //! Return whether compatible GPUs were found
+    bool haveCompatibleGpus() const;
+    //! Return a vector of handles, each to a device info for a compatible GPU.
+    std::vector<const DeviceInformation*> getDeviceInfos() const;
 };
 
 } // namespace test
index 8a90b7e710a17da6f0aa36b2c2c49591be3736e3..41d38eb4173666bfd677902da383e4543de0e07b 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -292,7 +292,7 @@ TYPED_TEST(HostAllocatorTestNoMem, Comparison)
 
 TYPED_TEST(HostAllocatorTestCopyable, TransfersWithPinningWorkWithCuda)
 {
-    if (!this->haveValidGpus())
+    if (!this->haveCompatibleGpus())
     {
         return;
     }
@@ -317,7 +317,7 @@ bool isPinned(const VectorType& v)
 
 TYPED_TEST(HostAllocatorTestCopyable, ManualPinningOperationsWorkWithCuda)
 {
-    if (!this->haveValidGpus())
+    if (!this->haveCompatibleGpus())
     {
         return;
     }
index 840aa1edb25876902ba9e523df4a6d9f04c4018f..3d518cb74b49e18ec5bac9875a50d053c72b8496 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -66,7 +66,7 @@ using PinnedMemoryCheckerTest = GpuTest;
 
 TEST_F(PinnedMemoryCheckerTest, DefaultContainerIsRecognized)
 {
-    if (!haveValidGpus())
+    if (!haveCompatibleGpus())
     {
         return;
     }
@@ -77,7 +77,7 @@ TEST_F(PinnedMemoryCheckerTest, DefaultContainerIsRecognized)
 
 TEST_F(PinnedMemoryCheckerTest, NonpinnedContainerIsRecognized)
 {
-    if (!haveValidGpus())
+    if (!haveCompatibleGpus())
     {
         return;
     }
@@ -89,7 +89,7 @@ TEST_F(PinnedMemoryCheckerTest, NonpinnedContainerIsRecognized)
 
 TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized)
 {
-    if (!haveValidGpus())
+    if (!haveCompatibleGpus())
     {
         return;
     }
@@ -101,7 +101,7 @@ TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized)
 
 TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized)
 {
-    if (!haveValidGpus())
+    if (!haveCompatibleGpus())
     {
         return;
     }
@@ -114,7 +114,7 @@ TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized)
 
 TEST_F(PinnedMemoryCheckerTest, PinnedCBufferIsRecognized)
 {
-    if (!haveValidGpus())
+    if (!haveCompatibleGpus())
     {
         return;
     }