Permit build with CUDA 11.0

The unannounced removal of SM 30 means cmake fails on all
GROMACS releases prior to this.

Fixes isHostMemoryPinned for changes in CUDA 11.0 and improves
test coverage.

Adapts tested and reviewed changes from !407 and !434

Refs #3631
Fixes #3632

diff --git a/cmake/gmxManageNvccConfig.cmake b/cmake/gmxManageNvccConfig.cmake
index 6f677d70d73c4694414f3f5337b03efb40e541b3..7a4afab2a7a4938be182515afe9aeb4062ef994d 100644 (file)
--- a/cmake/gmxManageNvccConfig.cmake
+++ b/cmake/gmxManageNvccConfig.cmake
@@ -103,9 +103,13 @@ else()
     #     => compile sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_61, sm_70 SASS, and compute_70 PTX
     # - with CUDA >=10.0        CC 7.5 is supported
     #     => compile sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_61, sm_70, sm_75 SASS, and compute_75 PTX
+    # - with CUDA >=11.0        CC 8.0 is supported
+    #     => compile sm_35, sm_37, sm_50, sm_52, sm_60, sm_61, sm_70, sm_75, sm_80 SASS, and compute_80 PTX
 
     # First add flags that trigger SASS (binary) code generation for physical arch
-    list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_30,code=sm_30")
+    if(CUDA_VERSION VERSION_LESS "11.0")
+       list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_30,code=sm_30")
+    endif()
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_35,code=sm_35")
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_37,code=sm_37")
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_50,code=sm_50")
@@ -113,6 +117,11 @@ else()
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_60,code=sm_60")
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_61,code=sm_61")
     list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_70,code=sm_70")
+    if(NOT CUDA_VERSION VERSION_LESS "11.0")
+        # Requesting sm or compute 35, 37, or 50 triggers deprecation messages with
+        # nvcc 11.0, which we need to suppress for use in CI
+        list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-Wno-deprecated-gpu-targets")
+    endif()
 
     # Next add flags that trigger PTX code generation for the newest supported virtual arch
     # that's useful to JIT to future architectures
@@ -125,6 +134,9 @@ else()
     if(NOT CUDA_VERSION VERSION_LESS "10.0")
         list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_75,code=compute_75")
     endif()
+    if(NOT CUDA_VERSION VERSION_LESS "11.0")
+        list (APPEND GMX_CUDA_NVCC_GENCODE_FLAGS "-gencode;arch=compute_80,code=compute_80")
+    endif()
 endif()
 
 if (GMX_CUDA_TARGET_SM)

diff --git a/docs/release-notes/2020/2020.4.rst b/docs/release-notes/2020/2020.4.rst
index 87eaf65e96d9bacfdda6afbb504db40842bcbaf4..155ef8efe4cebebc882a564a395dfe84ab1f3601 100644 (file)
--- a/docs/release-notes/2020/2020.4.rst
+++ b/docs/release-notes/2020/2020.4.rst
@@ -22,6 +22,15 @@ Fixes for ``gmx`` tools
 Fixes that affect portability
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+CUDA 11.0 supported
+^^^^^^^^^^^^^^^^^^^
+
+A build with CUDA 11.0 now configures and passes tests.
+Building with CUDA 11.0 means that hardware with CC 3.0 is no longer supported,
+while CC 8.0 can now be used.
+
+:issue:`3632`
+
 Miscellaneous
 ^^^^^^^^^^^^^
 

diff --git a/src/gromacs/gpu_utils/gpu_utils.cu b/src/gromacs/gpu_utils/gpu_utils.cu
index 216a930b67243c194feebecaf8948052d73ed397..6409e8b5e123da638c03df0a972a491d70448800 100644 (file)
--- a/src/gromacs/gpu_utils/gpu_utils.cu
+++ b/src/gromacs/gpu_utils/gpu_utils.cu
@@ -101,21 +101,34 @@ bool isHostMemoryPinned(const void* h_ptr)
     cudaPointerAttributes memoryAttributes;
     cudaError_t           stat = cudaPointerGetAttributes(&memoryAttributes, h_ptr);
 
-    bool result = false;
+    bool isPinned = false;
     switch (stat)
     {
-        case cudaSuccess: result = true; break;
+        case cudaSuccess:
+            // In CUDA 11.0, the field called memoryType in
+            // cudaPointerAttributes was replaced by a field called
+            // type, along with a documented change of behavior when the
+            // pointer passed to cudaPointerGetAttributes is to
+            // non-registered host memory. That change means that this
+            // code needs conditional compilation and different
+            // execution paths to function with all supported versions.
+#if CUDART_VERSION < 11 * 1000
+            isPinned = true;
+#else
+            isPinned = (memoryAttributes.type == cudaMemoryTypeHost);
+#endif
+            break;
 
         case cudaErrorInvalidValue:
             // If the buffer was not pinned, then it will not be recognized by CUDA at all
-            result = false;
+            isPinned = false;
             // Reset the last error status
             cudaGetLastError();
             break;
 
         default: CU_RET_ERR(stat, "Unexpected CUDA error");
     }
-    return result;
+    return isPinned;
 }
 
 /*!

diff --git a/src/gromacs/gpu_utils/tests/hostallocator.cpp b/src/gromacs/gpu_utils/tests/hostallocator.cpp
index 8a90b7e710a17da6f0aa36b2c2c49591be3736e3..4b4aeb36be64514148fc8a67c5e68f544c7c63e3 100644 (file)
--- a/src/gromacs/gpu_utils/tests/hostallocator.cpp
+++ b/src/gromacs/gpu_utils/tests/hostallocator.cpp
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -328,17 +328,17 @@ TYPED_TEST(HostAllocatorTestCopyable, ManualPinningOperationsWorkWithCuda)
     EXPECT_EQ(0, input.size());
     EXPECT_EQ(0, input.paddedSize());
     EXPECT_TRUE(input.empty());
-    EXPECT_FALSE(isPinned(input));
+    EXPECT_FALSE(isPinned(input)) << "should not be pinned before allocation";
 
     // Fill some contents, which will be pinned because of the policy.
     fillInput(&input, 1);
-    EXPECT_TRUE(isPinned(input));
+    EXPECT_TRUE(isPinned(input)) << "should be pinned after allocation";
 
     // Switching policy to CannotBePinned must unpin the buffer (via
     // realloc and copy).
     auto oldInputData = input.data();
     changePinningPolicy(&input, PinningPolicy::CannotBePinned);
-    EXPECT_FALSE(isPinned(input));
+    EXPECT_FALSE(isPinned(input)) << "should not be pinned after changing policy to CannotBePinned";
     // These cannot be equal as both had to be allocated at the same
     // time for the contents to be able to be copied.
     EXPECT_NE(oldInputData, input.data());
@@ -347,7 +347,7 @@ TYPED_TEST(HostAllocatorTestCopyable, ManualPinningOperationsWorkWithCuda)
     // realloc and copy).
     oldInputData = input.data();
     changePinningPolicy(&input, PinningPolicy::PinnedIfSupported);
-    EXPECT_TRUE(isPinned(input));
+    EXPECT_TRUE(isPinned(input)) << "should be pinned after changing policy to PinnedIfSupported";
     // These cannot be equal as both had to be allocated at the same
     // time for the contents to be able to be copied.
     EXPECT_NE(oldInputData, input.data());

diff --git a/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp b/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp
index 840aa1edb25876902ba9e523df4a6d9f04c4018f..305354943a68bec72958fcdcdc6c658f24fe016e 100644 (file)
--- a/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp
+++ b/src/gromacs/gpu_utils/tests/pinnedmemorychecker.cpp
@@ -1,7 +1,7 @@
 /*
  * This file is part of the GROMACS molecular simulation package.
  *
- * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2017,2018,2019,2020, by the GROMACS development team, led by
  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
  * and including many others, as listed in the AUTHORS file in the
  * top-level source directory and at http://www.gromacs.org.
@@ -99,6 +99,22 @@ TEST_F(PinnedMemoryCheckerTest, PinnedContainerIsRecognized)
     EXPECT_TRUE(isHostMemoryPinned(dummy.data()));
 }
 
+TEST_F(PinnedMemoryCheckerTest, PinningChangesAreRecognized)
+{
+    if (!haveValidGpus())
+    {
+        return;
+    }
+
+    HostVector<real> dummy(3, 1.5);
+    changePinningPolicy(&dummy, PinningPolicy::PinnedIfSupported);
+    EXPECT_TRUE(isHostMemoryPinned(dummy.data())) << "memory starts pinned";
+    changePinningPolicy(&dummy, PinningPolicy::CannotBePinned);
+    EXPECT_FALSE(isHostMemoryPinned(dummy.data())) << "memory is now unpinned";
+    changePinningPolicy(&dummy, PinningPolicy::PinnedIfSupported);
+    EXPECT_TRUE(isHostMemoryPinned(dummy.data())) << "memory is pinned again";
+}
+
 TEST_F(PinnedMemoryCheckerTest, DefaultCBufferIsRecognized)
 {
     if (!haveValidGpus())