Enable separate PME GPU rank

[alexxy/gromacs.git] / src / programs / mdrun / tests / pmetest.cpp

diff --git a/src/programs/mdrun/tests/pmetest.cpp b/src/programs/mdrun/tests/pmetest.cpp
index b91d71d543ce95310a1099cd2a6550642adad196..e1acdc8dee1c4940e0e01673959b7f3ebd58b422 100644 (file)
--- a/src/programs/mdrun/tests/pmetest.cpp
+++ b/src/programs/mdrun/tests/pmetest.cpp
@@ -34,10 +34,14 @@
  */
 /*! \internal \file
  * \brief
- * This implements basic PME sanity test (using single-rank mdrun).
+ * This implements basic PME sanity tests.
  * It runs the input system with PME for several steps (on CPU and GPU, if available),
  * and checks the reciprocal and conserved energies.
- * TODO: implement multi-rank tests as well.
+ * As part of mdrun-test, this will always run single rank PME simulation.
+ * As part of mdrun-mpi-test, this will run same as above when a single rank is requested,
+ * or a simulation with a single separate PME rank ("-npme 1") when multiple ranks are requested.
+ * \todo Extend and generalize this for more multi-rank tests (-npme 0, -npme 2, etc).
+ * \todo Implement death tests (e.g. for PME GPU decomposition).
  *
  * \author Aleksei Iupinov <a.yupinov@gmail.com>
  * \ingroup module_mdrun_integration_tests
@@ -50,11 +54,15 @@
 #include <string>
 #include <vector>
 
+#include <gtest/gtest-spi.h>
+
 #include "gromacs/gpu_utils/gpu_utils.h"
 #include "gromacs/hardware/gpu_hw_info.h"
 #include "gromacs/utility/cstringutil.h"
+#include "gromacs/utility/gmxmpi.h"
 #include "gromacs/utility/stringutil.h"
 
+#include "testutils/mpitest.h"
 #include "testutils/refdata.h"
 
 #include "energyreader.h"
@@ -111,6 +119,11 @@ TEST_F(PmeTest, ReproducesEnergies)
     const std::string inputFile = "spc-and-methanol";
     runner_.useTopGroAndNdxFromDatabase(inputFile.c_str());
 
+    // With single rank we can and will always test PP+PME as part of mdrun-test.
+    // With multiple ranks we can additionally test a single PME-only rank within mdrun-mpi-test.
+    const bool parallelRun    = (getNumberOfTestMpiRanks() > 1);
+    const bool useSeparatePme = parallelRun;
+
     EXPECT_EQ(0, runner_.callGrompp());
 
     //TODO test all proper/improper combinations in more thorough way?
@@ -122,7 +135,11 @@ TEST_F(PmeTest, ReproducesEnergies)
     runModes["PmeOnGpuFftAuto"]  = {"-pme", "gpu", "-pmefft", "auto"};
     TestReferenceData    refData;
     TestReferenceChecker rootChecker(refData.rootChecker());
-
+    const bool           thisRankChecks = (gmx_node_rank() == 0);
+    if (!thisRankChecks)
+    {
+        EXPECT_NONFATAL_FAILURE(rootChecker.checkUnusedEntries(), ""); // skip checks on other ranks
+    }
     for (const auto &mode : runModes)
     {
         auto modeTargetsGpus = (mode.first.find("Gpu") != std::string::npos);
@@ -138,30 +155,53 @@ TEST_F(PmeTest, ReproducesEnergies)
 
         CommandLine commandLine(mode.second);
         commandLine.append("-notunepme"); // for reciprocal energy reproducibility
+        if (useSeparatePme)
+        {
+            commandLine.addOption("-npme", 1);
+        }
         ASSERT_EQ(0, runner_.callMdrun(commandLine));
 
-        auto energyReader      = openEnergyFileToReadFields(runner_.edrFileName_, {"Coul. recip.", "Total Energy", "Kinetic En."});
-        auto conservedChecker  = rootChecker.checkCompound("Energy", "Conserved");
-        auto reciprocalChecker = rootChecker.checkCompound("Energy", "Reciprocal");
-        for (int i = 0; i <= nsteps; i++)
+        if (thisRankChecks)
         {
-            EnergyFrame frame            = energyReader->frame();
-            std::string stepNum          = gmx::formatString("%d", i);
-            const real  conservedEnergy  = frame.at("Total Energy");
-            const real  reciprocalEnergy = frame.at("Coul. recip.");
-            if (i == 0)
+            auto energyReader      = openEnergyFileToReadFields(runner_.edrFileName_, {"Coul. recip.", "Total Energy", "Kinetic En."});
+            auto conservedChecker  = rootChecker.checkCompound("Energy", "Conserved");
+            auto reciprocalChecker = rootChecker.checkCompound("Energy", "Reciprocal");
+            for (int i = 0; i <= nsteps; i++)
             {
-                // use first step values as references for tolerance
-                const real startingKineticEnergy = frame.at("Kinetic En.");
-                const auto conservedTolerance    = relativeToleranceAsFloatingPoint(startingKineticEnergy, 2e-5);
-                const auto reciprocalTolerance   = relativeToleranceAsFloatingPoint(reciprocalEnergy, 2e-5);
-                reciprocalChecker.setDefaultTolerance(reciprocalTolerance);
-                conservedChecker.setDefaultTolerance(conservedTolerance);
+                EnergyFrame frame            = energyReader->frame();
+                std::string stepNum          = gmx::formatString("%d", i);
+                const real  conservedEnergy  = frame.at("Total Energy");
+                const real  reciprocalEnergy = frame.at("Coul. recip.");
+                if (i == 0)
+                {
+                    // use first step values as references for tolerance
+                    const real startingKineticEnergy = frame.at("Kinetic En.");
+                    const auto conservedTolerance    = relativeToleranceAsFloatingPoint(startingKineticEnergy, 2e-5);
+                    const auto reciprocalTolerance   = relativeToleranceAsFloatingPoint(reciprocalEnergy, 3e-5);
+                    reciprocalChecker.setDefaultTolerance(reciprocalTolerance);
+                    conservedChecker.setDefaultTolerance(conservedTolerance);
+                }
+                conservedChecker.checkReal(conservedEnergy, stepNum.c_str());
+                reciprocalChecker.checkReal(reciprocalEnergy, stepNum.c_str());
             }
-            conservedChecker.checkReal(conservedEnergy, stepNum.c_str());
-            reciprocalChecker.checkReal(reciprocalEnergy, stepNum.c_str());
         }
+        // FIXME: without this barrier, one of the mdruns was somehow having a non-PME inputrec (!)
+#if GMX_LIB_MPI
+        if (parallelRun)
+        {
+            MPI_Barrier(MPI_COMM_WORLD);
+        }
+#endif
+    }
+
+    // This is a workaround for the output files to not be deleted in a parallel run.
+    // TODO: consider moving the barrier into the file manager destructor.
+#if GMX_LIB_MPI
+    if (parallelRun)
+    {
+        MPI_Barrier(MPI_COMM_WORLD);
     }
+#endif
 }
 
 }