/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2013,2014,2015,2016, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2018 by the GROMACS development team.
+ * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* \brief
* Tests for the mdrun multi-simulation functionality
*
- * \todo Test mdrun -multidir also
- *
* \author Mark Abraham <mark.j.abraham@gmail.com>
* \ingroup module_mdrun_integration_tests
*/
#include <gtest/gtest.h>
+#include "gromacs/mdtypes/md_enums.h"
#include "gromacs/utility/basenetwork.h"
+#include "gromacs/utility/gmxassert.h"
#include "gromacs/utility/path.h"
#include "gromacs/utility/real.h"
#include "gromacs/utility/stringutil.h"
#include "testutils/cmdlinetest.h"
+#include "testutils/mpitest.h"
#include "moduletest.h"
#include "terminationhelper.h"
namespace test
{
-MultiSimTest::MultiSimTest() : size_(gmx_node_num()),
- rank_(gmx_node_rank()),
- mdrunCaller_(new CommandLine)
+MultiSimTest::MultiSimTest() :
+ size_(gmx_node_num()),
+ rank_(gmx_node_rank()),
+ numRanksPerSimulation_(std::get<0>(GetParam())),
+ simulationNumber_(rank_ / numRanksPerSimulation_),
+ mdrunCaller_(new CommandLine)
+
{
- runner_.mdpInputFileName_ = fileManager_.getTemporaryFilePath(formatString("input%d.mdp", rank_));
- runner_.mdpOutputFileName_ = fileManager_.getTemporaryFilePath(formatString("output%d.mdp", rank_));
+ // Zero or less ranks doesn't make sense
+ GMX_RELEASE_ASSERT(numRanksPerSimulation_ > 0, "Invalid number of ranks per simulation.");
+
+ const char* directoryNameFormat = "sim_%d";
+
+ // Modify the file manager to have a temporary directory unique to
+ // each simulation. No need to have a mutex on this, nobody else
+ // can access the fileManager_ yet because we only just
+ // constructed it.
+ std::string originalTempDirectory = fileManager_.getOutputTempDirectory();
+ std::string newTempDirectory =
+ Path::join(originalTempDirectory, formatString(directoryNameFormat, simulationNumber_));
+ if (rank_ % numRanksPerSimulation_ == 0)
+ {
+ // Only one rank per simulation creates directory
+ Directory::create(newTempDirectory);
+ }
+#if GMX_LIB_MPI
+ // Make sure directories got created.
+ MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
+ fileManager_.setOutputTempDirectory(newTempDirectory);
- /* grompp needs to name the .tpr file so that when mdrun appends
- the MPI rank, it will find the right file. If we just used
- "%d.tpr" then \c TestFileManager prefixes that with an
- underscore. Then, there is no way for mdrun to be told the
- right name, because if you add the underscore manually, you get
- a second one from \c TestFileManager. However, it's easy to
- just start the suffix with "topol" in both cases. */
- runner_.tprFileName_ = fileManager_.getTemporaryFilePath(formatString("topol%d.tpr", rank_));
- mdrunTprFileName_ = fileManager_.getTemporaryFilePath("topol.tpr");
+ mdrunCaller_->append("mdrun");
+ mdrunCaller_->addOption("-multidir");
+ for (int i = 0; i < size_ / numRanksPerSimulation_; ++i)
+ {
+ mdrunCaller_->append(Path::join(originalTempDirectory, formatString(directoryNameFormat, i)));
+ }
+}
- runner_.useTopGroAndNdxFromDatabase("spc2");
+bool MultiSimTest::mpiSetupValid() const
+{
+ // Single simulation case is not implemented in multi-sim
+ const bool haveAtLeastTwoSimulations = ((size_ / numRanksPerSimulation_) >= 2);
+ // Mdrun will throw error if simulations don't have identical number of ranks
+ const bool simulationsHaveIdenticalRankNumber = ((size_ % numRanksPerSimulation_) == 0);
- mdrunCaller_->append("mdrun");
- mdrunCaller_->addOption("-multi", size_);
+ return (haveAtLeastTwoSimulations && simulationsHaveIdenticalRankNumber);
}
-void MultiSimTest::organizeMdpFile(const char *controlVariable,
- int numSteps)
+void MultiSimTest::organizeMdpFile(SimulationRunner* runner,
+ IntegrationAlgorithm integrator,
+ TemperatureCoupling tcoupl,
+ PressureCoupling pcoupl,
+ int numSteps,
+ bool doRegression) const
{
+ GMX_RELEASE_ASSERT(mpiSetupValid(), "Creating the mdp file without valid MPI setup is useless.");
const real baseTemperature = 298;
const real basePressure = 1;
- std::string mdpFileContents =
- formatString("nsteps = %d\n"
- "nstlog = 1\n"
- "nstcalcenergy = 1\n"
- "tcoupl = v-rescale\n"
- "tc-grps = System\n"
- "tau-t = 1\n"
- "ref-t = %f\n"
- // pressure coupling (if active)
- "tau-p = 1\n"
- "ref-p = %f\n"
- "compressibility = 4.5e-5\n"
- // velocity generation
- "gen-vel = yes\n"
- "gen-temp = %f\n"
- // control variable specification
- "%s\n",
- numSteps,
- baseTemperature + 0.0001*rank_,
- basePressure * std::pow(1.01, rank_),
- /* Set things up so that the initial KE decreases with
- increasing replica number, so that the (identical)
- starting PE decreases on the first step more for the
- replicas with higher number, which will tend to force
- replica exchange to occur. */
- std::max(baseTemperature - 10 * rank_, real(0)),
- controlVariable);
- runner_.useStringAsMdpFile(mdpFileContents);
+ std::string mdpFileContents = formatString(
+ "integrator = %s\n"
+ "tcoupl = %s\n"
+ "pcoupl = %s\n"
+ "nsteps = %d\n"
+ "nstlog = 1\n"
+ "nstcalcenergy = 1\n"
+ "tc-grps = System\n"
+ "tau-t = 1\n"
+ "ref-t = %f\n"
+ // pressure coupling (if active)
+ "tau-p = 2\n"
+ "ref-p = %f\n"
+ "compressibility = 4.5e-5\n"
+ // velocity generation
+ "gen-vel = yes\n"
+ "gen-temp = %f\n"
+ "gen-seed = %d\n"
+ // v-rescale and c-rescale use ld-seed
+ "ld-seed = %d\n"
+ // Two systems are used: spc2 non-interacting also at cutoff 1nm
+ // tip3p5 has box length of 1.86
+ "rcoulomb = 0.7\n"
+ "rvdw = 0.7\n"
+ // Trajectory output if required
+ "nstxout = %d\n"
+ "nstvout = %d\n"
+ "nstfout = %d\n"
+ "nstenergy = %d\n",
+ enumValueToString(integrator),
+ enumValueToString(tcoupl),
+ enumValueToString(pcoupl),
+ numSteps,
+ baseTemperature + 0.0001 * rank_,
+ basePressure * std::pow(1.01, rank_),
+ /* Set things up so that the initial KE decreases with
+ increasing replica number, so that the (identical)
+ starting PE decreases on the first step more for the
+ replicas with higher number, which will tend to force
+ replica exchange to occur. */
+ std::max(baseTemperature - 10 * rank_, real(0)),
+ // If we do regression, we need reproducible velocity
+ // generation, which can be different per simulation
+ (doRegression ? 671324 + simulationNumber_ : -1),
+ // If we do regression, we need reproducible temperature and
+ // pressure coupling, which can be different per simulation
+ (doRegression ? 51203 + simulationNumber_ : -1),
+ // If we do regression, write one intermediate point
+ (doRegression ? int(numSteps / 2) : 0),
+ (doRegression ? int(numSteps / 2) : 0),
+ (doRegression ? int(numSteps / 2) : 0),
+ // If we do regression, print energies every step so
+ // we're sure to catch the replica exchange steps
+ (doRegression ? 1 : 1000));
+ runner->useStringAsMdpFile(mdpFileContents);
+}
+
+void MultiSimTest::runGrompp(SimulationRunner* runner, int numSteps, bool doRegression, int maxWarnings) const
+{
+ // Call grompp once per simulation
+ if (rank_ % numRanksPerSimulation_ == 0)
+ {
+ const auto& simulator = std::get<1>(GetParam());
+ const auto& tcoupl = std::get<2>(GetParam());
+ const auto& pcoupl = std::get<3>(GetParam());
+ organizeMdpFile(runner, simulator, tcoupl, pcoupl, numSteps, doRegression);
+ CommandLine caller;
+ caller.addOption("-maxwarn", maxWarnings);
+ EXPECT_EQ(0, runner->callGromppOnThisRank(caller));
+ }
+
+#if GMX_LIB_MPI
+ // Make sure simulation masters have written the .tpr file before other ranks try to read it.
+ MPI_Barrier(MdrunTestFixtureBase::communicator_);
+#endif
}
void MultiSimTest::runExitsNormallyTest()
{
- if (size_ <= 1)
+ if (!mpiSetupValid())
{
- /* Can't test multi-sim without multiple ranks. */
+ // Can't test multi-sim without multiple simulations
return;
}
- const char *pcoupl = GetParam();
- organizeMdpFile(pcoupl);
- /* Call grompp on every rank - the standard callGrompp() only runs
- grompp on rank 0. */
- EXPECT_EQ(0, runner_.callGromppOnThisRank());
+ SimulationRunner runner(&fileManager_);
+ runner.useTopGroAndNdxFromDatabase("spc2");
+
+ runGrompp(&runner);
- // mdrun names the files without the rank suffix
- runner_.tprFileName_ = mdrunTprFileName_;
- ASSERT_EQ(0, runner_.callMdrun(*mdrunCaller_));
+ ASSERT_EQ(0, runner.callMdrun(*mdrunCaller_));
}
void MultiSimTest::runMaxhTest()
{
- if (size_ <= 1)
+ if (!mpiSetupValid())
{
- /* Can't test replica exchange without multiple ranks. */
+ // Can't test multi-sim without multiple simulations
return;
}
- TerminationHelper helper(&fileManager_, mdrunCaller_.get(), &runner_);
+ SimulationRunner runner(&fileManager_);
+ runner.useTopGroAndNdxFromDatabase("spc2");
+
+ TerminationHelper helper(&fileManager_, mdrunCaller_.get(), &runner);
// Make sure -maxh has a chance to propagate
- int numSteps = 100;
- organizeMdpFile("pcoupl = no", numSteps);
- /* Call grompp on every rank - the standard callGrompp() only runs
- grompp on rank 0. */
- EXPECT_EQ(0, runner_.callGromppOnThisRank());
-
- // mdrun names the files without the rank suffix
- runner_.tprFileName_ = mdrunTprFileName_;
-
- // The actual output checkpoint file gets a rank suffix, so
- // handle that in the expected result.
- std::string expectedCptFileName
- = Path::concatenateBeforeExtension(runner_.cptFileName_, formatString("%d", rank_));
- helper.runFirstMdrun(expectedCptFileName);
+ int numSteps = 100;
+ runGrompp(&runner, numSteps);
+
+ helper.runFirstMdrun(runner.cptFileName_);
helper.runSecondMdrun();
}
-} // namespace
-} // namespace
+} // namespace test
+} // namespace gmx
biod.pnpi.spb.ru / alexxy / gromacs.git / blobdiff