/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2013,2014,2015, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2018 by the GROMACS development team.
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* \brief
* Tests for the mdrun multi-simulation functionality
*
- * \todo Test mdrun -multidir also
- *
* \author Mark Abraham <mark.j.abraham@gmail.com>
- * \ingroup module_mdrun
+ * \ingroup module_mdrun_integration_tests
*/
#include "gmxpre.h"
#include <gtest/gtest.h>
+#include "gromacs/mdtypes/md_enums.h"
#include "gromacs/utility/basenetwork.h"
+#include "gromacs/utility/gmxassert.h"
+#include "gromacs/utility/path.h"
#include "gromacs/utility/real.h"
#include "gromacs/utility/stringutil.h"
#include "testutils/cmdlinetest.h"
+#include "testutils/mpitest.h"
#include "moduletest.h"
+#include "terminationhelper.h"
namespace gmx
{
namespace test
{
-MultiSimTest::MultiSimTest() : size_(gmx_node_num()),
- rank_(gmx_node_rank()),
- mdrunCaller_(new CommandLine)
+MultiSimTest::MultiSimTest() :
+ size_(gmx_node_num()),
+ rank_(gmx_node_rank()),
+ numRanksPerSimulation_(std::get<0>(GetParam())),
+ simulationNumber_(rank_ / numRanksPerSimulation_),
+ mdrunCaller_(new CommandLine)
+
{
- runner_.mdpInputFileName_ = fileManager_.getTemporaryFilePath(formatString("input%d.mdp", rank_));
- runner_.mdpOutputFileName_ = fileManager_.getTemporaryFilePath(formatString("output%d.mdp", rank_));
-
- /* grompp needs to name the .tpr file so that when mdrun appends
- the MPI rank, it will find the right file. If we just used
- "%d.tpr" then \c TestFileManager prefixes that with an
- underscore. Then, there is no way for mdrun to be told the
- right name, because if you add the underscore manually, you get
- a second one from \c TestFileManager. However, it's easy to
- just start the suffix with "topol" in both cases. */
- runner_.tprFileName_ = fileManager_.getTemporaryFilePath(formatString("topol%d.tpr", rank_));
- mdrunTprFileName_ = fileManager_.getTemporaryFilePath("topol.tpr");
-
- runner_.useTopGroAndNdxFromDatabase("spc2");
-
- mdrunCaller_->append("mdrun_mpi");
- mdrunCaller_->addOption("-multi", size_);
+ // Zero or less ranks doesn't make sense
+ GMX_RELEASE_ASSERT(numRanksPerSimulation_ > 0, "Invalid number of ranks per simulation.");
+
+ const char* directoryNameFormat = "sim_%d";
+
+ // Modify the file manager to have a temporary directory unique to
+ // each simulation. No need to have a mutex on this, nobody else
+ // can access the fileManager_ yet because we only just
+ // constructed it.
+ std::string originalTempDirectory = fileManager_.getOutputTempDirectory();
+ std::string newTempDirectory =
+ Path::join(originalTempDirectory, formatString(directoryNameFormat, simulationNumber_));
+ if (rank_ % numRanksPerSimulation_ == 0)
+ {
+ // Only one rank per simulation creates directory
+ Directory::create(newTempDirectory);
+ }
+#if GMX_LIB_MPI
+ // Make sure directories got created.
+ MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
+ fileManager_.setOutputTempDirectory(newTempDirectory);
+
+ mdrunCaller_->append("mdrun");
+ mdrunCaller_->addOption("-multidir");
+ for (int i = 0; i < size_ / numRanksPerSimulation_; ++i)
+ {
+ mdrunCaller_->append(Path::join(originalTempDirectory, formatString(directoryNameFormat, i)));
+ }
}
-void MultiSimTest::organizeMdpFile(const char *controlVariable)
+bool MultiSimTest::mpiSetupValid() const
{
+ // Single simulation case is not implemented in multi-sim
+ const bool haveAtLeastTwoSimulations = ((size_ / numRanksPerSimulation_) >= 2);
+ // Mdrun will throw error if simulations don't have identical number of ranks
+ const bool simulationsHaveIdenticalRankNumber = ((size_ % numRanksPerSimulation_) == 0);
+
+ return (haveAtLeastTwoSimulations && simulationsHaveIdenticalRankNumber);
+}
+
+void MultiSimTest::organizeMdpFile(SimulationRunner* runner,
+ IntegrationAlgorithm integrator,
+ TemperatureCoupling tcoupl,
+ PressureCoupling pcoupl,
+ int numSteps,
+ bool doRegression) const
+{
+ GMX_RELEASE_ASSERT(mpiSetupValid(), "Creating the mdp file without valid MPI setup is useless.");
const real baseTemperature = 298;
const real basePressure = 1;
- std::string mdpFileContents =
- formatString("nsteps = 2\n"
- "nstlog = 1\n"
- "nstcalcenergy = 1\n"
- "tcoupl = v-rescale\n"
- "tc-grps = System\n"
- "tau-t = 1\n"
- "ref-t = %f\n"
- // pressure coupling (if active)
- "tau-p = 1\n"
- "ref-p = %f\n"
- "compressibility = 4.5e-5\n"
- // velocity generation
- "gen-vel = yes\n"
- "gen-temp = %f\n"
- // control variable specification
- "%s\n",
- baseTemperature + 0.0001*rank_,
- basePressure * std::pow(1.01, rank_),
- /* Set things up so that the initial KE decreases with
- increasing replica number, so that the (identical)
- starting PE decreases on the first step more for the
- replicas with higher number, which will tend to force
- replica exchange to occur. */
- std::max(baseTemperature - 10 * rank_, real(0)),
- controlVariable);
- runner_.useStringAsMdpFile(mdpFileContents);
+ std::string mdpFileContents = formatString(
+ "integrator = %s\n"
+ "tcoupl = %s\n"
+ "pcoupl = %s\n"
+ "nsteps = %d\n"
+ "nstlog = 1\n"
+ "nstcalcenergy = 1\n"
+ "tc-grps = System\n"
+ "tau-t = 1\n"
+ "ref-t = %f\n"
+ // pressure coupling (if active)
+ "tau-p = 2\n"
+ "ref-p = %f\n"
+ "compressibility = 4.5e-5\n"
+ // velocity generation
+ "gen-vel = yes\n"
+ "gen-temp = %f\n"
+ "gen-seed = %d\n"
+ // v-rescale and c-rescale use ld-seed
+ "ld-seed = %d\n"
+ // Two systems are used: spc2 non-interacting also at cutoff 1nm
+ // tip3p5 has box length of 1.86
+ "rcoulomb = 0.7\n"
+ "rvdw = 0.7\n"
+ // Trajectory output if required
+ "nstxout = %d\n"
+ "nstvout = %d\n"
+ "nstfout = %d\n"
+ "nstenergy = %d\n",
+ enumValueToString(integrator),
+ enumValueToString(tcoupl),
+ enumValueToString(pcoupl),
+ numSteps,
+ baseTemperature + 0.0001 * rank_,
+ basePressure * std::pow(1.01, rank_),
+ /* Set things up so that the initial KE decreases with
+ increasing replica number, so that the (identical)
+ starting PE decreases on the first step more for the
+ replicas with higher number, which will tend to force
+ replica exchange to occur. */
+ std::max(baseTemperature - 10 * rank_, real(0)),
+ // If we do regression, we need reproducible velocity
+ // generation, which can be different per simulation
+ (doRegression ? 671324 + simulationNumber_ : -1),
+ // If we do regression, we need reproducible temperature and
+ // pressure coupling, which can be different per simulation
+ (doRegression ? 51203 + simulationNumber_ : -1),
+ // If we do regression, write one intermediate point
+ (doRegression ? int(numSteps / 2) : 0),
+ (doRegression ? int(numSteps / 2) : 0),
+ (doRegression ? int(numSteps / 2) : 0),
+ // If we do regression, print energies every step so
+ // we're sure to catch the replica exchange steps
+ (doRegression ? 1 : 1000));
+ runner->useStringAsMdpFile(mdpFileContents);
+}
+
+void MultiSimTest::runGrompp(SimulationRunner* runner, int numSteps, bool doRegression, int maxWarnings) const
+{
+ // Call grompp once per simulation
+ if (rank_ % numRanksPerSimulation_ == 0)
+ {
+ const auto& simulator = std::get<1>(GetParam());
+ const auto& tcoupl = std::get<2>(GetParam());
+ const auto& pcoupl = std::get<3>(GetParam());
+ organizeMdpFile(runner, simulator, tcoupl, pcoupl, numSteps, doRegression);
+ CommandLine caller;
+ caller.addOption("-maxwarn", maxWarnings);
+ EXPECT_EQ(0, runner->callGromppOnThisRank(caller));
+ }
+
+#if GMX_LIB_MPI
+ // Make sure simulation masters have written the .tpr file before other ranks try to read it.
+ MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
+}
+
+void MultiSimTest::runExitsNormallyTest()
+{
+ if (!mpiSetupValid())
+ {
+ // Can't test multi-sim without multiple simulations
+ return;
+ }
+
+ SimulationRunner runner(&fileManager_);
+ runner.useTopGroAndNdxFromDatabase("spc2");
+
+ runGrompp(&runner);
+
+ ASSERT_EQ(0, runner.callMdrun(*mdrunCaller_));
+}
+
+void MultiSimTest::runMaxhTest()
+{
+ if (!mpiSetupValid())
+ {
+ // Can't test multi-sim without multiple simulations
+ return;
+ }
+
+ SimulationRunner runner(&fileManager_);
+ runner.useTopGroAndNdxFromDatabase("spc2");
+
+ TerminationHelper helper(&fileManager_, mdrunCaller_.get(), &runner);
+ // Make sure -maxh has a chance to propagate
+ int numSteps = 100;
+ runGrompp(&runner, numSteps);
+
+ helper.runFirstMdrun(runner.cptFileName_);
+ helper.runSecondMdrun();
}
-} // namespace
-} // namespace
+} // namespace test
+} // namespace gmx