Bug fix for irregular replex steps with GPU update
authorAlan Gray <alangray3@gmail.com>
Mon, 11 Oct 2021 12:42:49 +0000 (12:42 +0000)
committerMark Abraham <mark.j.abraham@gmail.com>
Mon, 11 Oct 2021 12:42:49 +0000 (12:42 +0000)
src/gromacs/mdrun/md.cpp
src/programs/mdrun/tests/CMakeLists.txt
src/programs/mdrun/tests/periodicactions.cpp
src/programs/mdrun/tests/periodicactions.h [new file with mode: 0644]
src/programs/mdrun/tests/periodicactions_basic.cpp [new file with mode: 0644]
src/programs/mdrun/tests/periodicactions_constraints.cpp [new file with mode: 0644]
src/programs/mdrun/tests/periodicactions_coupling.cpp [new file with mode: 0644]
src/programs/mdrun/tests/replicaexchange.cpp
src/programs/mdrun/tests/replicaexchange_equivalence.cpp [new file with mode: 0644]

index cd657319c00c8e95426c52e21527c526dbd11646..4f6f0263f27a5faab674c33718711b7148cbfeda 100644 (file)
@@ -922,15 +922,15 @@ void gmx::LegacySimulator::do_md()
         do_verbose = mdrunOptions.verbose
                      && (step % mdrunOptions.verboseStepPrintInterval == 0 || bFirstStep || bLastStep);
 
-        if (useGpuForUpdate && !bFirstStep && bNS)
+        // On search steps, when doing the update on the GPU, copy
+        // the coordinates and velocities to the host unless they are
+        // already there (ie on the first step and after replica
+        // exchange).
+        if (useGpuForUpdate && bNS && !bFirstStep && !bExchanged)
         {
-            // Copy velocities from the GPU on search steps to keep a copy on host (device buffers are reinitialized).
             stateGpu->copyVelocitiesFromGpu(state->v, AtomLocality::Local);
-            stateGpu->waitVelocitiesReadyOnHost(AtomLocality::Local);
-            // Copy coordinate from the GPU when needed at the search step.
-            // NOTE: The cases when coordinates needed on CPU for force evaluation are handled in sim_utils.
-            // NOTE: If the coordinates are to be written into output file they are also copied separately before the output.
             stateGpu->copyCoordinatesFromGpu(state->x, AtomLocality::Local);
+            stateGpu->waitVelocitiesReadyOnHost(AtomLocality::Local);
             stateGpu->waitCoordinatesReadyOnHost(AtomLocality::Local);
         }
 
@@ -962,13 +962,15 @@ void gmx::LegacySimulator::do_md()
                 if (correct_box(fplog, step, state->box))
                 {
                     bMasterState = TRUE;
-                    // If update is offloaded, it should be informed about the box size change
-                    if (useGpuForUpdate)
-                    {
-                        integrator->setPbc(PbcType::Xyz, state->box);
-                    }
                 }
             }
+            // If update is offloaded, and the box was changed either
+            // above or in a replica exchange on the previous step,
+            // the GPU Update object should be informed
+            if (useGpuForUpdate && (bMasterState || bExchanged))
+            {
+                integrator->setPbc(PbcType::Xyz, state->box);
+            }
             if (haveDDAtomOrdering(*cr) && bMasterState)
             {
                 dd_collect_state(cr->dd, state, state_global);
@@ -1489,7 +1491,8 @@ void gmx::LegacySimulator::do_md()
         {
             if (useGpuForUpdate)
             {
-                if (bNS && (bFirstStep || haveDDAtomOrdering(*cr)))
+                // On search steps, update handles to device vectors
+                if (bNS && (bFirstStep || haveDDAtomOrdering(*cr) || bExchanged))
                 {
                     integrator->set(stateGpu->getCoordinates(),
                                     stateGpu->getVelocities(),
@@ -1501,8 +1504,9 @@ void gmx::LegacySimulator::do_md()
                     /* The velocity copy is redundant if we had Center-of-Mass motion removed on
                      * the previous step. We don't check that now. */
                     stateGpu->copyVelocitiesToGpu(state->v, AtomLocality::Local);
-                    if (!runScheduleWork->stepWork.haveGpuPmeOnThisRank
-                        && !runScheduleWork->stepWork.useGpuXBufferOps)
+                    if (bExchanged
+                        || (!runScheduleWork->stepWork.haveGpuPmeOnThisRank
+                            && !runScheduleWork->stepWork.useGpuXBufferOps))
                     {
                         stateGpu->copyCoordinatesToGpu(state->x, AtomLocality::Local);
                     }
@@ -1533,15 +1537,6 @@ void gmx::LegacySimulator::do_md()
                                       doParrinelloRahman,
                                       ir->nstpcouple * ir->delta_t,
                                       M);
-
-                // Copy velocities D2H after update if:
-                // - Globals are computed this step (includes the energy output steps).
-                // - Temperature is needed for the next step.
-                if (bGStat || needHalfStepKineticEnergy)
-                {
-                    stateGpu->copyVelocitiesFromGpu(state->v, AtomLocality::Local);
-                    stateGpu->waitVelocitiesReadyOnHost(AtomLocality::Local);
-                }
             }
             else
             {
@@ -1640,14 +1635,33 @@ void gmx::LegacySimulator::do_md()
             // and when algorithms require it.
             const bool doInterSimSignal = (simulationsShareState && do_per_step(step, nstSignalComm));
 
-            if (bGStat || needHalfStepKineticEnergy || doInterSimSignal)
+            if (useGpuForUpdate)
             {
-                // Copy coordinates when needed to stop the CM motion.
-                if (useGpuForUpdate && (bDoReplEx || (!EI_VV(ir->eI) && bStopCM)))
+                const bool coordinatesRequiredForStopCM =
+                        bStopCM && (bGStat || needHalfStepKineticEnergy || doInterSimSignal)
+                        && !EI_VV(ir->eI);
+
+                // Copy coordinates when needed to stop the CM motion or for replica exchange
+                if (coordinatesRequiredForStopCM || bDoReplEx)
                 {
                     stateGpu->copyCoordinatesFromGpu(state->x, AtomLocality::Local);
                     stateGpu->waitCoordinatesReadyOnHost(AtomLocality::Local);
                 }
+
+                // Copy velocities back to the host if:
+                // - Globals are computed this step (includes the energy output steps).
+                // - Temperature is needed for the next step.
+                // - This is a replica exchange step (even though we will only need
+                //     the velocities if an exchange succeeds)
+                if (bGStat || needHalfStepKineticEnergy || bDoReplEx)
+                {
+                    stateGpu->copyVelocitiesFromGpu(state->v, AtomLocality::Local);
+                    stateGpu->waitVelocitiesReadyOnHost(AtomLocality::Local);
+                }
+            }
+
+            if (bGStat || needHalfStepKineticEnergy || doInterSimSignal)
+            {
                 // Since we're already communicating at this step, we
                 // can propagate intra-simulation signals. Note that
                 // check_nstglobalcomm has the responsibility for
@@ -1722,6 +1736,7 @@ void gmx::LegacySimulator::do_md()
             accumulateKineticLambdaComponents(enerd, state->lambda, *ir->fepvals);
         }
 
+        bool scaleCoordinates = !useGpuForUpdate || bDoReplEx;
         update_pcouple_after_coordinates(fplog,
                                          step,
                                          ir,
@@ -1735,7 +1750,7 @@ void gmx::LegacySimulator::do_md()
                                          state,
                                          nrnb,
                                          upd.deform(),
-                                         !useGpuForUpdate);
+                                         scaleCoordinates);
 
         const bool doBerendsenPressureCoupling = (inputrec->epc == PressureCoupling::Berendsen
                                                   && do_per_step(step, inputrec->nstpcouple));
index 32ff54143f39454872f6e83069493d201774a6a2..e80d1647f8f0db09fd8f39f8e31199132e37c107 100644 (file)
@@ -190,6 +190,19 @@ gmx_add_gtest_executable(${exename} MPI
         # files with code for tests
         multisim.cpp
         multisimtest.cpp
+        # pseudo-library for code for mdrun
+        $<TARGET_OBJECTS:mdrun_objlib>
+        )
+target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
+gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 4 INTEGRATION_TEST IGNORE_LEAKS)
+
+set(testname "MdrunMultiSimReplexTests")
+set(exename "mdrun-multisim-replex-test")
+
+gmx_add_gtest_executable(${exename} MPI
+    CPP_SOURCE_FILES
+        # files with code for tests
+       multisimtest.cpp
         replicaexchange.cpp
         # pseudo-library for code for mdrun
         $<TARGET_OBJECTS:mdrun_objlib>
@@ -197,6 +210,20 @@ gmx_add_gtest_executable(${exename} MPI
 target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
 gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 4 INTEGRATION_TEST IGNORE_LEAKS)
 
+set(testname "MdrunMultiSimReplexEquivalenceTests")
+set(exename "mdrun-multisim-replex-equivalence-test")
+
+gmx_add_gtest_executable(${exename} MPI
+    CPP_SOURCE_FILES
+        # files with code for tests
+       multisimtest.cpp
+        replicaexchange_equivalence.cpp
+        # pseudo-library for code for mdrun
+        $<TARGET_OBJECTS:mdrun_objlib>
+        )
+target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
+gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 4 INTEGRATION_TEST IGNORE_LEAKS)
+
 # Tests that only make sense to run with multiple ranks and/or real
 # MPI are implemented here. Special case for slow PME tests
 set(testname "MdrunMpiPmeTests")
@@ -213,23 +240,60 @@ target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
 gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 2 OPENMP_THREADS 2 INTEGRATION_TEST IGNORE_LEAKS)
 
 # Slow-running tests that target testing multiple-rank coordination behaviors
-set(exename "mdrun-mpi-coordination-test")
+# These tests are extremely slow without optimization or OpenMP, so only run them for
+# build types like Release or RelWithDebInfo and if the build has been configured
+# with OpenMP enabled
+set(exename "mdrun-mpi-coordination-basic-test")
 gmx_add_gtest_executable(${exename} MPI
     CPP_SOURCE_FILES
         # files with code for tests
         periodicactions.cpp
+       periodicactions_basic.cpp
+        # pseudo-library for code for mdrun
+        $<TARGET_OBJECTS:mdrun_objlib>
+        )
+target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
+
+if (CMAKE_BUILD_TYPE MATCHES "Rel" AND GMX_OPENMP)
+    set(testname "MdrunMpiCoordinationBasicTestsOneRank")
+    gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 1 SLOW_TEST IGNORE_LEAKS)
+    set(testname "MdrunMpiCoordinationBasicTestsTwoRanks")
+    gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 2 SLOW_TEST IGNORE_LEAKS)
+endif()
+
+set(exename "mdrun-mpi-coordination-coupling-test")
+gmx_add_gtest_executable(${exename} MPI
+    CPP_SOURCE_FILES
+        # files with code for tests
+        periodicactions.cpp
+       periodicactions_coupling.cpp
+        # pseudo-library for code for mdrun
+        $<TARGET_OBJECTS:mdrun_objlib>
+        )
+target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
+
+if (CMAKE_BUILD_TYPE MATCHES "Rel" AND GMX_OPENMP)
+    set(testname "MdrunMpiCoordinationCouplingTestsOneRank")
+    gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 1 SLOW_TEST IGNORE_LEAKS)
+    set(testname "MdrunMpiCoordinationCouplingTestsTwoRanks")
+    gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 2 SLOW_TEST IGNORE_LEAKS)
+endif()
+
+set(exename "mdrun-mpi-coordination-constraints-test")
+gmx_add_gtest_executable(${exename} MPI
+    CPP_SOURCE_FILES
+        # files with code for tests
+        periodicactions.cpp
+       periodicactions_constraints.cpp
         # pseudo-library for code for mdrun
         $<TARGET_OBJECTS:mdrun_objlib>
         )
 target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
 
-# These tests are extremely slow without optimization or OpenMP, so only run them for
-# build types like Release or RelWithDebInfo and if the build has been configured
-# with OpenMP enabled.
 if (CMAKE_BUILD_TYPE MATCHES "Rel" AND GMX_OPENMP)
-    set(testname "MdrunMpiCoordinationTestsOneRank")
+    set(testname "MdrunMpiCoordinationConstraintsTestsOneRank")
     gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 1 SLOW_TEST IGNORE_LEAKS)
-    set(testname "MdrunMpiCoordinationTestsTwoRanks")
+    set(testname "MdrunMpiCoordinationConstraintsTestsTwoRanks")
     gmx_register_gtest_test(${testname} ${exename} MPI_RANKS 2 SLOW_TEST IGNORE_LEAKS)
 endif()
 
index c0868398243fa4911db09766e5e34dd33bc82ecf..6308abc2c31bb6f686d4742db9a8b33329013492 100644 (file)
@@ -34,7 +34,7 @@
  */
 
 /*! \internal \file
- * \brief Tests to verify that a simulator that only does some actions
+ * \brief Utility functions for tests to verify that a simulator that only does some actions
  * periodically produces the expected results.
  *
  * \author Mark Abraham <mark.j.abraham@gmail.com>
 
 #include "config.h"
 
-#include <tuple>
-
-#include "gromacs/trajectory/energyframe.h"
-#include "gromacs/utility/strconvert.h"
-#include "gromacs/utility/stringutil.h"
-
-#include "testutils/simulationdatabase.h"
-#include "testutils/testasserts.h"
-
-#include "energycomparison.h"
-#include "energyreader.h"
-#include "moduletest.h"
+#include "periodicactions.h"
 
 namespace gmx
 {
 namespace test
 {
-namespace
-{
 
 /*! \brief Mdp parameters that determine the manner of simulation
  * propagation. */
@@ -72,46 +59,9 @@ using PropagationParameters = MdpFieldValues;
  *  not the simulation propagation. */
 using PeriodicOutputParameters = MdpFieldValues;
 
-//! Helper type of output file names for the reference mdrun call
-struct ReferenceFileNames
-{
-    //! Name of energy file
-    std::string edrFileName_;
-};
-
 //! Function type to produce sets of .mdp parameters for testing periodic output
 using OutputParameterGeneratorFunction = std::vector<PeriodicOutputParameters> (*)();
 
-/*! \brief Test fixture base for comparing a simulator with one that
- * does everything every step
- *
- * This test ensures that two simulator code paths called via
- * different mdp options yield identical energy trajectories,
- * up to some (arbitrary) precision.
- *
- * These tests are useful to check that periodic actions implemented
- * in simulators are correct, and that different code paths expected
- * to yield identical results are equivalent.
- */
-class PeriodicActionsTest :
-    public MdrunTestFixture,
-    public ::testing::WithParamInterface<std::tuple<PropagationParameters, OutputParameterGeneratorFunction>>
-{
-public:
-    // PeriodicActionsTest();
-    //! Run mdrun with given output parameters
-    void doMdrun(const PeriodicOutputParameters& output);
-    //! Generate reference data from mdrun writing everything every step.
-    void prepareReferenceData();
-    //! Names for the output files from the reference mdrun call
-    ReferenceFileNames referenceFileNames_ = { fileManager_.getTemporaryFilePath("reference.edr") };
-    //! Functor for energy comparison
-    EnergyComparison energyComparison_{ EnergyComparison::defaultEnergyTermsToCompare(),
-                                        MaxNumFrames::compareAllFrames() };
-    //! Names of energies compared by energyComparison_
-    std::vector<std::string> namesOfEnergiesToMatch_ = energyComparison_.getEnergyNames();
-};
-
 void PeriodicActionsTest::doMdrun(const PeriodicOutputParameters& output)
 {
     auto propagation = std::get<0>(GetParam());
@@ -262,21 +212,13 @@ TEST_P(PeriodicActionsTest, PeriodicActionsAgreeWithReference)
 
 /*! \brief Some common choices of periodic output mdp parameters to
  * simplify defining values for the combinations under test */
-PeriodicOutputParameters g_basicPeriodicOutputParameters = {
+static PeriodicOutputParameters g_basicPeriodicOutputParameters = {
     { "nstenergy", "0" }, { "nstlog", "0" },           { "nstxout", "0" },
     { "nstvout", "0" },   { "nstfout", "0" },          { "nstxout-compressed", "0" },
     { "nstdhdl", "0" },   { "description", "unknown" }
 };
 
-/*! \brief Return vector of mdp parameter sets to test
- *
- * These are constructed to observe the mdp parameter choices that
- * only affect when output is written agree with those that were
- * written from a reference run where output was done every step. The
- * numbers are chosen in the context of the defaults in
- * prepareDefaultMdpFieldValues().
- *
- * \todo Test nstlog, nstdhdl, nstxout-compressed */
+// \todo Test nstlog, nstdhdl, nstxout-compressed
 std::vector<PeriodicOutputParameters> outputParameters()
 {
     std::vector<PeriodicOutputParameters> parameterSets;
@@ -304,7 +246,6 @@ std::vector<PeriodicOutputParameters> outputParameters()
     return parameterSets;
 }
 
-//! Returns sets of simple simulation propagators
 std::vector<PropagationParameters> simplePropagationParameters()
 {
     return {
@@ -320,14 +261,6 @@ std::vector<PropagationParameters> simplePropagationParameters()
     };
 }
 
-/*! \brief Returns sets of simulation propagators including coupling
- *
- * These are chosen to cover the commonly used space of propagation
- * algorithms togther with the perdiods between their actions. The
- * periods tested are chosen to be mutually co-prime and distinct from
- * the pair search and user output period (4), so that over the
- * duration of a short simulation many kinds of simulation step
- * behavior are tested. */
 std::vector<PropagationParameters> propagationParametersWithCoupling()
 {
     std::string nsttcouple = "2";
@@ -394,10 +327,6 @@ std::vector<PropagationParameters> propagationParametersWithCoupling()
     return parameterSets;
 }
 
-/*! \brief Returns sets of simulation propagators including coupling
- *
- * These are chosen to cover the commonly used space of propagation
- * algorithms on systems with constraints. */
 std::vector<PropagationParameters> propagationParametersWithConstraints()
 {
     std::string nsttcouple = "2";
@@ -460,37 +389,5 @@ std::vector<PropagationParameters> propagationParametersWithConstraints()
     return parameterSets;
 }
 
-using ::testing::Combine;
-using ::testing::Values;
-using ::testing::ValuesIn;
-
-// TODO The time for OpenCL kernel compilation means these tests time
-// out. Once that compilation is cached for the whole process, these
-// tests can run in such configurations.
-#if !GMX_GPU_OPENCL
-INSTANTIATE_TEST_SUITE_P(BasicPropagators,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(simplePropagationParameters()), Values(outputParameters)));
-INSTANTIATE_TEST_SUITE_P(PropagatorsWithCoupling,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(propagationParametersWithCoupling()), Values(outputParameters)));
-INSTANTIATE_TEST_SUITE_P(PropagatorsWithConstraints,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(propagationParametersWithConstraints()),
-                                 Values(outputParameters)));
-#else
-INSTANTIATE_TEST_SUITE_P(DISABLED_BasicPropagators,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(simplePropagationParameters()), Values(outputParameters)));
-INSTANTIATE_TEST_SUITE_P(DISABLED_PropagatorsWithCoupling,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(propagationParametersWithCoupling()), Values(outputParameters)));
-INSTANTIATE_TEST_SUITE_P(DISABLED_PropagatorsWithConstraints,
-                         PeriodicActionsTest,
-                         Combine(ValuesIn(propagationParametersWithConstraints()),
-                                 Values(outputParameters)));
-#endif
-
-} // namespace
 } // namespace test
 } // namespace gmx
diff --git a/src/programs/mdrun/tests/periodicactions.h b/src/programs/mdrun/tests/periodicactions.h
new file mode 100644 (file)
index 0000000..e965562
--- /dev/null
@@ -0,0 +1,144 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief Interfaces of related classes for tests to verify that a simulator that only does some
+ * actions periodically produces the expected results.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \ingroup module_mdrun_integration_tests
+ */
+#include "gmxpre.h"
+
+#include "config.h"
+
+#include <tuple>
+
+#include "gromacs/trajectory/energyframe.h"
+#include "gromacs/utility/strconvert.h"
+#include "gromacs/utility/stringutil.h"
+
+#include "testutils/simulationdatabase.h"
+#include "testutils/testasserts.h"
+
+#include "energycomparison.h"
+#include "energyreader.h"
+#include "moduletest.h"
+
+namespace gmx
+{
+namespace test
+{
+
+/*! \brief Mdp parameters that determine the manner of simulation
+ * propagation. */
+using PropagationParameters = MdpFieldValues;
+
+/*! \brief Mdp parameters that should only affect the observations,
+ *  not the simulation propagation. */
+using PeriodicOutputParameters = MdpFieldValues;
+
+//! Helper type of output file names for the reference mdrun call
+struct ReferenceFileNames
+{
+    //! Name of energy file
+    std::string edrFileName_;
+};
+
+//! Function type to produce sets of .mdp parameters for testing periodic output
+using OutputParameterGeneratorFunction = std::vector<PeriodicOutputParameters> (*)();
+
+/*! \brief Test fixture base for comparing a simulator with one that
+ * does everything every step
+ *
+ * This test ensures that two simulator code paths called via
+ * different mdp options yield identical energy trajectories,
+ * up to some (arbitrary) precision.
+ *
+ * These tests are useful to check that periodic actions implemented
+ * in simulators are correct, and that different code paths expected
+ * to yield identical results are equivalent.
+ */
+class PeriodicActionsTest :
+    public MdrunTestFixture,
+    public ::testing::WithParamInterface<std::tuple<PropagationParameters, OutputParameterGeneratorFunction>>
+{
+public:
+    // PeriodicActionsTest();
+    //! Run mdrun with given output parameters
+    void doMdrun(const PeriodicOutputParameters& output);
+    //! Generate reference data from mdrun writing everything every step.
+    void prepareReferenceData();
+    //! Names for the output files from the reference mdrun call
+    ReferenceFileNames referenceFileNames_ = { fileManager_.getTemporaryFilePath("reference.edr") };
+    //! Functor for energy comparison
+    EnergyComparison energyComparison_{ EnergyComparison::defaultEnergyTermsToCompare(),
+                                        MaxNumFrames::compareAllFrames() };
+    //! Names of energies compared by energyComparison_
+    std::vector<std::string> namesOfEnergiesToMatch_ = energyComparison_.getEnergyNames();
+};
+
+/*! \brief Return vector of mdp parameter sets to test
+ *
+ * These are constructed to observe the mdp parameter choices that
+ * only affect when output is written agree with those that were
+ * written from a reference run where output was done every step. The
+ * numbers are chosen in the context of the defaults in
+ * prepareDefaultMdpFieldValues().
+ *
+ * \todo Test nstlog, nstdhdl, nstxout-compressed */
+std::vector<PeriodicOutputParameters> outputParameters();
+
+//! Returns sets of simple simulation propagators
+std::vector<PropagationParameters> simplePropagationParameters();
+
+/*! \brief Returns sets of simulation propagators including coupling
+ *
+ * These are chosen to cover the commonly used space of propagation
+ * algorithms togther with the perdiods between their actions. The
+ * periods tested are chosen to be mutually co-prime and distinct from
+ * the pair search and user output period (4), so that over the
+ * duration of a short simulation many kinds of simulation step
+ * behavior are tested. */
+std::vector<PropagationParameters> propagationParametersWithCoupling();
+
+/*! \brief Returns sets of simulation propagators including coupling
+ *
+ * These are chosen to cover the commonly used space of propagation
+ * algorithms on systems with constraints. */
+std::vector<PropagationParameters> propagationParametersWithConstraints();
+
+} // namespace test
+} // namespace gmx
diff --git a/src/programs/mdrun/tests/periodicactions_basic.cpp b/src/programs/mdrun/tests/periodicactions_basic.cpp
new file mode 100644 (file)
index 0000000..6ee44a2
--- /dev/null
@@ -0,0 +1,72 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief Tests to verify that a simulator that only does some actions
+ * periodically with basic propagators produces the expected results.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \ingroup module_mdrun_integration_tests
+ */
+#include "gmxpre.h"
+
+#include "config.h"
+
+#include "periodicactions.h"
+
+namespace gmx
+{
+namespace test
+{
+
+using ::testing::Combine;
+using ::testing::Values;
+using ::testing::ValuesIn;
+
+// TODO The time for OpenCL kernel compilation means these tests time
+// out. Once that compilation is cached for the whole process, these
+// tests can run in such configurations.
+#if !GMX_GPU_OPENCL
+INSTANTIATE_TEST_SUITE_P(BasicPropagators,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(simplePropagationParameters()), Values(outputParameters)));
+#else
+INSTANTIATE_TEST_SUITE_P(DISABLED_BasicPropagators,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(simplePropagationParameters()), Values(outputParameters)));
+#endif
+
+} // namespace test
+} // namespace gmx
diff --git a/src/programs/mdrun/tests/periodicactions_constraints.cpp b/src/programs/mdrun/tests/periodicactions_constraints.cpp
new file mode 100644 (file)
index 0000000..038a8f2
--- /dev/null
@@ -0,0 +1,74 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief Tests to verify that a simulator that only does some actions
+ * periodically with propagators with constraints produces the expected results.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \ingroup module_mdrun_integration_tests
+ */
+#include "gmxpre.h"
+
+#include "config.h"
+
+#include "periodicactions.h"
+
+namespace gmx
+{
+namespace test
+{
+
+using ::testing::Combine;
+using ::testing::Values;
+using ::testing::ValuesIn;
+
+// TODO The time for OpenCL kernel compilation means these tests time
+// out. Once that compilation is cached for the whole process, these
+// tests can run in such configurations.
+#if !GMX_GPU_OPENCL
+INSTANTIATE_TEST_SUITE_P(PropagatorsWithConstraints,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(propagationParametersWithConstraints()),
+                                 Values(outputParameters)));
+#else
+INSTANTIATE_TEST_SUITE_P(DISABLED_PropagatorsWithConstraints,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(propagationParametersWithConstraints()),
+                                 Values(outputParameters)));
+#endif
+
+} // namespace test
+} // namespace gmx
diff --git a/src/programs/mdrun/tests/periodicactions_coupling.cpp b/src/programs/mdrun/tests/periodicactions_coupling.cpp
new file mode 100644 (file)
index 0000000..5a442d2
--- /dev/null
@@ -0,0 +1,72 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief Tests to verify that a simulator that only does some actions
+ * periodically with propagators with coupling produces the expected results.
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \ingroup module_mdrun_integration_tests
+ */
+#include "gmxpre.h"
+
+#include "config.h"
+
+#include "periodicactions.h"
+
+namespace gmx
+{
+namespace test
+{
+
+using ::testing::Combine;
+using ::testing::Values;
+using ::testing::ValuesIn;
+
+// TODO The time for OpenCL kernel compilation means these tests time
+// out. Once that compilation is cached for the whole process, these
+// tests can run in such configurations.
+#if !GMX_GPU_OPENCL
+INSTANTIATE_TEST_SUITE_P(PropagatorsWithCoupling,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(propagationParametersWithCoupling()), Values(outputParameters)));
+#else
+INSTANTIATE_TEST_SUITE_P(DISABLED_PropagatorsWithCoupling,
+                         PeriodicActionsTest,
+                         Combine(ValuesIn(propagationParametersWithCoupling()), Values(outputParameters)));
+#endif
+
+} // namespace test
+} // namespace gmx
index 41d70455960912833f393cb703bcca9af426403b..f363c0e7dc627346287a5dd86fbc4f7e80085c24 100644 (file)
@@ -114,243 +114,5 @@ INSTANTIATE_TEST_SUITE_P(InNvt,
                                             ::testing::Values(TemperatureCoupling::VRescale),
                                             ::testing::Values(PressureCoupling::No)));
 
-/*! \brief Return replica exchange related output from logfile
- *
- * All replica exchange related output in log files start with 'Repl',
- * making extraction easy. This function also removes the printing of
- * energy differences, as the log files are compared exactly, and
- * energy differences will slightly vary between runs.
- *
- * \param logFileName  Name of log file
- * \return  Replica exchange related output in log file
- */
-static std::string getReplicaExchangeOutputFromLogFile(const std::string& logFileName)
-{
-    TextInputFile logFile(logFileName);
-    std::string   replExOutput;
-    std::string   line;
-    while (logFile.readLine(&line))
-    {
-        // All replica exchange output lines starts with "Repl"
-        if (startsWith(line, "Repl"))
-        {
-            // This is an exact comparison, so we can't compare the energies which
-            // are slightly different per run. Energies are tested later.
-            const auto pos = line.find("dE_term");
-            if (pos != std::string::npos)
-            {
-                line.replace(line.begin() + pos, line.end(), "[ not checked ]\n");
-            }
-            replExOutput.append(line);
-        }
-    }
-    return replExOutput;
-}
-
-//! Convenience typedef
-typedef MultiSimTest ReplicaExchangeRegressionTest;
-
-/* Run replica exchange simulations, compare to reference data
- *
- * Reference data generated by
- *
- * GROMACS version:    2022-dev
- * Precision:          single and double (separate reference data)
- * Memory model:       64 bit
- * MPI library:        MPI
- * OpenMP support:     enabled (GMX_OPENMP_MAX_THREADS = 64)
- * GPU support:        disabled
- * SIMD instructions:  AVX2_256
- * FFT library:        fftw-3.3.9-sse2-avx
- * RDTSCP usage:       enabled
- * TNG support:        enabled
- * Hwloc support:      disabled
- * Tracing support:    disabled
- * C compiler:         /usr/local/bin/mpicc Clang 8.0.1
- * C++ compiler:       /usr/local/bin/mpic++ Clang 8.0.1
- *
- */
-TEST_P(ReplicaExchangeRegressionTest, WithinTolerances)
-{
-    if (!mpiSetupValid())
-    {
-        // Can't test multi-sim without multiple simulations
-        return;
-    }
-
-    if (size_ != 4)
-    {
-        // Results are depending on number of ranks, and we can't have reference
-        // data for all cases. Restricting the regression tests to runs with 4 ranks.
-        // This allows testing 4 replicas with single rank, or 2 replicas with 2 ranks each.
-        return;
-    }
-    const auto& tcoupl = std::get<2>(GetParam());
-    const auto& pcoupl = std::get<3>(GetParam());
-
-    const int numSteps       = 16;
-    const int exchangePeriod = 4;
-    // grompp warns about generating velocities and using parrinello-rahman
-    const int maxWarnings = (pcoupl == PressureCoupling::ParrinelloRahman ? 1 : 0);
-
-    mdrunCaller_->addOption("-replex", exchangePeriod);
-    // Seeds need to be reproducible for regression, but can be different per simulation
-    mdrunCaller_->addOption("-reseed", 98713 + simulationNumber_);
-
-    SimulationRunner runner(&fileManager_);
-    runner.useTopGroAndNdxFromDatabase("tip3p5");
-
-    runGrompp(&runner, numSteps, true, maxWarnings);
-    ASSERT_EQ(0, runner.callMdrun(*mdrunCaller_));
-
-#if GMX_LIB_MPI
-    // Make sure all simulations are finished before checking the results.
-    MPI_Barrier(MdrunTestFixtureBase::communicator_);
-#endif
-
-    // We only test simulation results on one rank to avoid problems with reference file access.
-    if (rank_ == 0)
-    {
-        // Create reference data helper object
-        TestReferenceData refData;
-
-        // Specify how energy trajectory comparison must work
-        const auto hasConservedField =
-                !(tcoupl == TemperatureCoupling::No && pcoupl == PressureCoupling::No);
-        // Tolerances copied from simulator tests
-        EnergyTermsToCompare energyTermsToCompare{ {
-                { interaction_function[F_EPOT].longname,
-                  relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
-                { interaction_function[F_EKIN].longname,
-                  relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
-        } };
-        if (hasConservedField)
-        {
-            energyTermsToCompare.emplace(interaction_function[F_ECONSERVED].longname,
-                                         relativeToleranceAsPrecisionDependentUlp(50.0, 100, 80));
-        }
-        if (pcoupl != PressureCoupling::No)
-        {
-            energyTermsToCompare.emplace("Volume",
-                                         relativeToleranceAsPrecisionDependentUlp(10.0, 200, 160));
-        }
-
-        // Specify how trajectory frame matching must work.
-        const TrajectoryFrameMatchSettings trajectoryMatchSettings{ true,
-                                                                    true,
-                                                                    true,
-                                                                    ComparisonConditions::MustCompare,
-                                                                    ComparisonConditions::MustCompare,
-                                                                    ComparisonConditions::MustCompare,
-                                                                    MaxNumFrames::compareAllFrames() };
-        TrajectoryTolerances trajectoryTolerances = TrajectoryComparison::s_defaultTrajectoryTolerances;
-        // By default, velocity tolerance is MUCH tighter than force tolerance
-        trajectoryTolerances.velocities = trajectoryTolerances.forces;
-        // Build the functor that will compare reference and test
-        // trajectory frames in the chosen way.
-        TrajectoryComparison trajectoryComparison{ trajectoryMatchSettings, trajectoryTolerances };
-
-        // Loop over simulations
-        for (int simulationNumber = 0; simulationNumber < (size_ / numRanksPerSimulation_);
-             simulationNumber++)
-        {
-            TestReferenceChecker simulationChecker(refData.rootChecker().checkCompound(
-                    "Simulation", formatString("Replica %d", simulationNumber)));
-
-            const auto logFileName =
-                    std::regex_replace(runner.logFileName_,
-                                       std::regex(formatString("sim_%d", simulationNumber_)),
-                                       formatString("sim_%d", simulationNumber));
-            const auto energyFileName =
-                    std::regex_replace(runner.edrFileName_,
-                                       std::regex(formatString("sim_%d", simulationNumber_)),
-                                       formatString("sim_%d", simulationNumber));
-            const auto trajectoryFileName =
-                    std::regex_replace(runner.fullPrecisionTrajectoryFileName_,
-                                       std::regex(formatString("sim_%d", simulationNumber_)),
-                                       formatString("sim_%d", simulationNumber));
-
-            // Check log replica exchange related output (contains exchange statistics)
-            auto replicaExchangeOutputChecker =
-                    simulationChecker.checkCompound("ReplExOutput", "Output");
-            const auto replExOutput = getReplicaExchangeOutputFromLogFile(logFileName);
-            replicaExchangeOutputChecker.checkTextBlock(replExOutput, "Replica Exchange Output");
-
-            // Check that the energies agree with the refdata within tolerance.
-            checkEnergiesAgainstReferenceData(energyFileName, energyTermsToCompare, &simulationChecker);
-
-            // Check that the trajectories agree with the refdata within tolerance.
-            checkTrajectoryAgainstReferenceData(trajectoryFileName, trajectoryComparison, &simulationChecker);
-
-        } // end loop over simulations
-    }     // end testing simulations on one rank
-
-#if GMX_LIB_MPI
-    // Make sure testing is complete before returning - ranks delete temporary files on exit
-    MPI_Barrier(MdrunTestFixtureBase::communicator_);
-#endif
-}
-
-/*! \brief Helper struct printing custom test name
- *
- * Regression test results not only depend on the test parameters, but
- * also on the total number of ranks and the precision. Names must
- * reflect that to identify correct reference data.
- */
-struct PrintReplicaExchangeParametersToString
-{
-    template<class ParamType>
-    std::string operator()(const testing::TestParamInfo<ParamType>& parameter) const
-    {
-        auto testIdentifier =
-                formatString("ReplExRegression_%s_%s_%s_%dRanks_%dRanksPerSimulation_%s",
-                             enumValueToString(std::get<1>(parameter.param)),
-                             enumValueToString(std::get<2>(parameter.param)),
-                             enumValueToString(std::get<3>(parameter.param)),
-                             gmx_node_num(),
-                             static_cast<int>(std::get<0>(parameter.param)),
-                             GMX_DOUBLE ? "d" : "s");
-        // Valid GTest names cannot include hyphens
-        testIdentifier.erase(std::remove(testIdentifier.begin(), testIdentifier.end(), '-'),
-                             testIdentifier.end());
-        return testIdentifier;
-    }
-};
-
-#if GMX_LIB_MPI
-INSTANTIATE_TEST_SUITE_P(
-        ReplicaExchangeIsEquivalentToReferenceLeapFrog,
-        ReplicaExchangeRegressionTest,
-        ::testing::Combine(::testing::Values(NumRanksPerSimulation(1), NumRanksPerSimulation(2)),
-                           ::testing::Values(IntegrationAlgorithm::MD),
-                           ::testing::Values(TemperatureCoupling::VRescale, TemperatureCoupling::NoseHoover),
-                           ::testing::Values(PressureCoupling::CRescale, PressureCoupling::ParrinelloRahman)),
-        PrintReplicaExchangeParametersToString());
-INSTANTIATE_TEST_SUITE_P(ReplicaExchangeIsEquivalentToReferenceVelocityVerlet,
-                         ReplicaExchangeRegressionTest,
-                         ::testing::Combine(::testing::Values(NumRanksPerSimulation(1),
-                                                              NumRanksPerSimulation(2)),
-                                            ::testing::Values(IntegrationAlgorithm::VV),
-                                            ::testing::Values(TemperatureCoupling::NoseHoover),
-                                            ::testing::Values(PressureCoupling::No)),
-                         PrintReplicaExchangeParametersToString());
-#else
-INSTANTIATE_TEST_SUITE_P(
-        DISABLED_ReplicaExchangeIsEquivalentToReferenceLeapFrog,
-        ReplicaExchangeRegressionTest,
-        ::testing::Combine(::testing::Values(NumRanksPerSimulation(1), NumRanksPerSimulation(2)),
-                           ::testing::Values(IntegrationAlgorithm::MD),
-                           ::testing::Values(TemperatureCoupling::VRescale, TemperatureCoupling::NoseHoover),
-                           ::testing::Values(PressureCoupling::CRescale, PressureCoupling::ParrinelloRahman)),
-        PrintReplicaExchangeParametersToString());
-INSTANTIATE_TEST_SUITE_P(DISABLED_ReplicaExchangeIsEquivalentToReferenceVelocityVerlet,
-                         ReplicaExchangeRegressionTest,
-                         ::testing::Combine(::testing::Values(NumRanksPerSimulation(1),
-                                                              NumRanksPerSimulation(2)),
-                                            ::testing::Values(IntegrationAlgorithm::VV),
-                                            ::testing::Values(TemperatureCoupling::NoseHoover),
-                                            ::testing::Values(PressureCoupling::No)),
-                         PrintReplicaExchangeParametersToString());
-#endif
 } // namespace test
 } // namespace gmx
diff --git a/src/programs/mdrun/tests/replicaexchange_equivalence.cpp b/src/programs/mdrun/tests/replicaexchange_equivalence.cpp
new file mode 100644 (file)
index 0000000..a3d7872
--- /dev/null
@@ -0,0 +1,310 @@
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2013,2014,2015,2016,2018 by the GROMACS development team.
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief
+ * Equivalence tests for the mdrun replica-exchange functionality
+ *
+ * \author Mark Abraham <mark.j.abraham@gmail.com>
+ * \ingroup module_mdrun_integration_tests
+ */
+#include "gmxpre.h"
+
+#include "config.h"
+
+#include <regex>
+
+#include <gtest/gtest.h>
+
+#include "gromacs/mdtypes/md_enums.h"
+#include "gromacs/topology/ifunc.h"
+#include "gromacs/utility/basenetwork.h"
+#include "gromacs/utility/filestream.h"
+#include "gromacs/utility/path.h"
+#include "gromacs/utility/stringutil.h"
+
+#include "testutils/refdata.h"
+#include "testutils/testfilemanager.h"
+
+#include "energycomparison.h"
+#include "multisimtest.h"
+#include "trajectorycomparison.h"
+
+namespace gmx
+{
+namespace test
+{
+
+/*! \brief Return replica exchange related output from logfile
+ *
+ * All replica exchange related output in log files start with 'Repl',
+ * making extraction easy. This function also removes the printing of
+ * energy differences, as the log files are compared exactly, and
+ * energy differences will slightly vary between runs.
+ *
+ * \param logFileName  Name of log file
+ * \return  Replica exchange related output in log file
+ */
+static std::string getReplicaExchangeOutputFromLogFile(const std::string& logFileName)
+{
+    TextInputFile logFile(logFileName);
+    std::string   replExOutput;
+    std::string   line;
+    while (logFile.readLine(&line))
+    {
+        // All replica exchange output lines starts with "Repl"
+        if (startsWith(line, "Repl"))
+        {
+            // This is an exact comparison, so we can't compare the energies which
+            // are slightly different per run. Energies are tested later.
+            const auto pos = line.find("dE_term");
+            if (pos != std::string::npos)
+            {
+                line.replace(line.begin() + pos, line.end(), "[ not checked ]\n");
+            }
+            replExOutput.append(line);
+        }
+    }
+    return replExOutput;
+}
+
+//! Convenience typedef
+typedef MultiSimTest ReplicaExchangeRegressionTest;
+
+/* Run replica exchange simulations, compare to reference data
+ *
+ * Reference data generated by
+ *
+ * GROMACS version:    2022-dev
+ * Precision:          single and double (separate reference data)
+ * Memory model:       64 bit
+ * MPI library:        MPI
+ * OpenMP support:     enabled (GMX_OPENMP_MAX_THREADS = 64)
+ * GPU support:        disabled
+ * SIMD instructions:  AVX2_256
+ * FFT library:        fftw-3.3.9-sse2-avx
+ * RDTSCP usage:       enabled
+ * TNG support:        enabled
+ * Hwloc support:      disabled
+ * Tracing support:    disabled
+ * C compiler:         /usr/local/bin/mpicc Clang 8.0.1
+ * C++ compiler:       /usr/local/bin/mpic++ Clang 8.0.1
+ *
+ */
+TEST_P(ReplicaExchangeRegressionTest, WithinTolerances)
+{
+    if (!mpiSetupValid())
+    {
+        // Can't test multi-sim without multiple simulations
+        return;
+    }
+
+    if (size_ != 4)
+    {
+        // Results are depending on number of ranks, and we can't have reference
+        // data for all cases. Restricting the regression tests to runs with 4 ranks.
+        // This allows testing 4 replicas with single rank, or 2 replicas with 2 ranks each.
+        return;
+    }
+    const auto& tcoupl = std::get<2>(GetParam());
+    const auto& pcoupl = std::get<3>(GetParam());
+
+    const int numSteps       = 16;
+    const int exchangePeriod = 4;
+    // grompp warns about generating velocities and using parrinello-rahman
+    const int maxWarnings = (pcoupl == PressureCoupling::ParrinelloRahman ? 1 : 0);
+
+    mdrunCaller_->addOption("-replex", exchangePeriod);
+    // Seeds need to be reproducible for regression, but can be different per simulation
+    mdrunCaller_->addOption("-reseed", 98713 + simulationNumber_);
+
+    SimulationRunner runner(&fileManager_);
+    runner.useTopGroAndNdxFromDatabase("tip3p5");
+
+    runGrompp(&runner, numSteps, true, maxWarnings);
+    ASSERT_EQ(0, runner.callMdrun(*mdrunCaller_));
+
+#if GMX_LIB_MPI
+    // Make sure all simulations are finished before checking the results.
+    MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
+
+    // We only test simulation results on one rank to avoid problems with reference file access.
+    if (rank_ == 0)
+    {
+        // Create reference data helper object
+        TestReferenceData refData;
+
+        // Specify how energy trajectory comparison must work
+        const auto hasConservedField =
+                !(tcoupl == TemperatureCoupling::No && pcoupl == PressureCoupling::No);
+        // Tolerances copied from simulator tests
+        EnergyTermsToCompare energyTermsToCompare{ {
+                { interaction_function[F_EPOT].longname,
+                  relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
+                { interaction_function[F_EKIN].longname,
+                  relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
+        } };
+        if (hasConservedField)
+        {
+            energyTermsToCompare.emplace(interaction_function[F_ECONSERVED].longname,
+                                         relativeToleranceAsPrecisionDependentUlp(50.0, 100, 80));
+        }
+        if (pcoupl != PressureCoupling::No)
+        {
+            energyTermsToCompare.emplace("Volume",
+                                         relativeToleranceAsPrecisionDependentUlp(10.0, 200, 160));
+        }
+
+        // Specify how trajectory frame matching must work.
+        const TrajectoryFrameMatchSettings trajectoryMatchSettings{ true,
+                                                                    true,
+                                                                    true,
+                                                                    ComparisonConditions::MustCompare,
+                                                                    ComparisonConditions::MustCompare,
+                                                                    ComparisonConditions::MustCompare,
+                                                                    MaxNumFrames::compareAllFrames() };
+        TrajectoryTolerances trajectoryTolerances = TrajectoryComparison::s_defaultTrajectoryTolerances;
+        // By default, velocity tolerance is MUCH tighter than force tolerance
+        trajectoryTolerances.velocities = trajectoryTolerances.forces;
+        // Build the functor that will compare reference and test
+        // trajectory frames in the chosen way.
+        TrajectoryComparison trajectoryComparison{ trajectoryMatchSettings, trajectoryTolerances };
+
+        // Loop over simulations
+        for (int simulationNumber = 0; simulationNumber < (size_ / numRanksPerSimulation_);
+             simulationNumber++)
+        {
+            TestReferenceChecker simulationChecker(refData.rootChecker().checkCompound(
+                    "Simulation", formatString("Replica %d", simulationNumber)));
+
+            const auto logFileName =
+                    std::regex_replace(runner.logFileName_,
+                                       std::regex(formatString("sim_%d", simulationNumber_)),
+                                       formatString("sim_%d", simulationNumber));
+            const auto energyFileName =
+                    std::regex_replace(runner.edrFileName_,
+                                       std::regex(formatString("sim_%d", simulationNumber_)),
+                                       formatString("sim_%d", simulationNumber));
+            const auto trajectoryFileName =
+                    std::regex_replace(runner.fullPrecisionTrajectoryFileName_,
+                                       std::regex(formatString("sim_%d", simulationNumber_)),
+                                       formatString("sim_%d", simulationNumber));
+
+            // Check log replica exchange related output (contains exchange statistics)
+            auto replicaExchangeOutputChecker =
+                    simulationChecker.checkCompound("ReplExOutput", "Output");
+            const auto replExOutput = getReplicaExchangeOutputFromLogFile(logFileName);
+            replicaExchangeOutputChecker.checkTextBlock(replExOutput, "Replica Exchange Output");
+
+            // Check that the energies agree with the refdata within tolerance.
+            checkEnergiesAgainstReferenceData(energyFileName, energyTermsToCompare, &simulationChecker);
+
+            // Check that the trajectories agree with the refdata within tolerance.
+            checkTrajectoryAgainstReferenceData(trajectoryFileName, trajectoryComparison, &simulationChecker);
+
+        } // end loop over simulations
+    }     // end testing simulations on one rank
+
+#if GMX_LIB_MPI
+    // Make sure testing is complete before returning - ranks delete temporary files on exit
+    MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
+}
+
+/*! \brief Helper struct printing custom test name
+ *
+ * Regression test results not only depend on the test parameters, but
+ * also on the total number of ranks and the precision. Names must
+ * reflect that to identify correct reference data.
+ */
+struct PrintReplicaExchangeParametersToString
+{
+    template<class ParamType>
+    std::string operator()(const testing::TestParamInfo<ParamType>& parameter) const
+    {
+        auto testIdentifier =
+                formatString("ReplExRegression_%s_%s_%s_%dRanks_%dRanksPerSimulation_%s",
+                             enumValueToString(std::get<1>(parameter.param)),
+                             enumValueToString(std::get<2>(parameter.param)),
+                             enumValueToString(std::get<3>(parameter.param)),
+                             gmx_node_num(),
+                             static_cast<int>(std::get<0>(parameter.param)),
+                             GMX_DOUBLE ? "d" : "s");
+        // Valid GTest names cannot include hyphens
+        testIdentifier.erase(std::remove(testIdentifier.begin(), testIdentifier.end(), '-'),
+                             testIdentifier.end());
+        return testIdentifier;
+    }
+};
+
+#if GMX_LIB_MPI
+INSTANTIATE_TEST_SUITE_P(
+        ReplicaExchangeIsEquivalentToReferenceLeapFrog,
+        ReplicaExchangeRegressionTest,
+        ::testing::Combine(::testing::Values(NumRanksPerSimulation(1), NumRanksPerSimulation(2)),
+                           ::testing::Values(IntegrationAlgorithm::MD),
+                           ::testing::Values(TemperatureCoupling::VRescale, TemperatureCoupling::NoseHoover),
+                           ::testing::Values(PressureCoupling::CRescale, PressureCoupling::ParrinelloRahman)),
+        PrintReplicaExchangeParametersToString());
+INSTANTIATE_TEST_SUITE_P(ReplicaExchangeIsEquivalentToReferenceVelocityVerlet,
+                         ReplicaExchangeRegressionTest,
+                         ::testing::Combine(::testing::Values(NumRanksPerSimulation(1),
+                                                              NumRanksPerSimulation(2)),
+                                            ::testing::Values(IntegrationAlgorithm::VV),
+                                            ::testing::Values(TemperatureCoupling::NoseHoover),
+                                            ::testing::Values(PressureCoupling::No)),
+                         PrintReplicaExchangeParametersToString());
+#else
+INSTANTIATE_TEST_SUITE_P(
+        DISABLED_ReplicaExchangeIsEquivalentToReferenceLeapFrog,
+        ReplicaExchangeRegressionTest,
+        ::testing::Combine(::testing::Values(NumRanksPerSimulation(1), NumRanksPerSimulation(2)),
+                           ::testing::Values(IntegrationAlgorithm::MD),
+                           ::testing::Values(TemperatureCoupling::VRescale, TemperatureCoupling::NoseHoover),
+                           ::testing::Values(PressureCoupling::CRescale, PressureCoupling::ParrinelloRahman)),
+        PrintReplicaExchangeParametersToString());
+INSTANTIATE_TEST_SUITE_P(DISABLED_ReplicaExchangeIsEquivalentToReferenceVelocityVerlet,
+                         ReplicaExchangeRegressionTest,
+                         ::testing::Combine(::testing::Values(NumRanksPerSimulation(1),
+                                                              NumRanksPerSimulation(2)),
+                                            ::testing::Values(IntegrationAlgorithm::VV),
+                                            ::testing::Values(TemperatureCoupling::NoseHoover),
+                                            ::testing::Values(PressureCoupling::No)),
+                         PrintReplicaExchangeParametersToString());
+#endif
+} // namespace test
+} // namespace gmx