Merge branch release-2021
authorMark Abraham <mark.j.abraham@gmail.com>
Fri, 18 Dec 2020 07:23:07 +0000 (08:23 +0100)
committerMark Abraham <mark.j.abraham@gmail.com>
Fri, 18 Dec 2020 07:41:23 +0000 (08:41 +0100)
Some reformatting changes needed for the difference in
.clang-format in the two branches

34 files changed:
1  2 
CMakeLists.txt
admin/gitlab-ci/global.gitlab-ci.yml
admin/gitlab-ci/gromacs.gitlab-ci.yml
admin/gitlab-ci/python-gmxapi02.gitlab-ci.yml
admin/gitlab-ci/python-gmxapi03.gitlab-ci.yml
api/CMakeLists.txt
api/nblib/gmxsetup.cpp
cmake/gmxVersionInfo.cmake
docs/release-notes/2021/major/deprecated-functionality.rst
python_packaging/src/setup.py
src/gromacs/ewald/pme_load_balancing.cpp
src/gromacs/fileio/checkpoint.cpp
src/gromacs/fileio/tests/CMakeLists.txt
src/gromacs/gmxlib/nonbonded/nb_free_energy.cpp
src/gromacs/mdlib/forcerec.cpp
src/gromacs/mdrun/runner.cpp
src/gromacs/mdtypes/checkpointdata.h
src/gromacs/modularsimulator/checkpointhelper.h
src/gromacs/modularsimulator/energydata.cpp
src/gromacs/modularsimulator/freeenergyperturbationdata.cpp
src/gromacs/modularsimulator/modularsimulator.cpp
src/gromacs/modularsimulator/statepropagatordata.cpp
src/gromacs/nbnxm/benchmark/bench_setup.cpp
src/gromacs/nbnxm/cuda/nbnxm_cuda_kernel.cuh
src/gromacs/nbnxm/opencl/nbnxm_ocl_kernel.clh
src/gromacs/simd/impl_arm_sve/impl_arm_sve_simd_double.h
src/gromacs/simd/impl_arm_sve/impl_arm_sve_simd_float.h
src/gromacs/utility/keyvaluetree.cpp
src/programs/CMakeLists.txt
src/programs/mdrun/tests/CMakeLists.txt
src/programs/mdrun/tests/checkpoint.cpp
src/programs/mdrun/tests/exactcontinuation.cpp
src/programs/mdrun/tests/freeenergy.cpp
src/programs/mdrun/tests/simulator.cpp

diff --cc CMakeLists.txt
Simple merge
Simple merge
index 939b51142de68461cbb6563f1d59f6281b1f45dc,582be7e014e1b42975c6a9f44199a8000722be8c..b55d9fd323e63939c4701e9311d547dcca35d54f
@@@ -230,8 -230,8 +230,8 @@@ gromacs:gcc-10:configure
    variables:
      CMAKE: /usr/local/cmake-3.13.0/bin/cmake
      CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=AVX2_256"
 -    CMAKE_EXTRA_OPTIONS: "-DGMX_EXTERNAL_CLFFT=ON"
 +    CMAKE_EXTRA_OPTIONS: "-DGMX_EXTERNAL_CLFFT=ON -DGMX_INSTALL_LEGACY_API=ON"
-     COMPILER_MAJOR_VERSION: 7
+     COMPILER_MAJOR_VERSION: 10
  
  gromacs:clang-8-cuda-10.0:configure:
    extends:
index 8f0588cdb6ac220337345e214ed16c22dd8fbc2b,0000000000000000000000000000000000000000..fc267ecfa7e993db009f40ed381bf6662686e633
mode 100644,000000..100644
--- /dev/null
@@@ -1,61 -1,0 +1,61 @@@
- .gmxapi-0.2:gcc-7:gmx2021:
 +#
 +# Jobs to test gmxapi client (Python) packages
 +#
 +
-   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-gcc-7
++.gmxapi-0.2:gcc-10:gmx2021:
 +  extends:
 +    - .variables:default
 +    - .use-clang:base
-     - job: gromacs:gcc-7:build
++  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-10
 +  stage: test
 +  variables:
 +    KUBERNETES_CPU_LIMIT: 2
 +    KUBERNETES_CPU_REQUEST: 2
 +    KUBERNETES_MEMORY_LIMIT: 2Gi
 +    KUBERNETES_MEMORY_REQUEST: 2Gi
 +    PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml
 +    PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml
 +    PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml
 +    PY_MPI_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-mpi-JUnitTestResults.xml
 +  script:
 +    - source $INSTALL_DIR/bin/GMXRC
 +    - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh
 +  artifacts:
 +    reports:
 +      junit:
 +        - $PY_UNIT_TEST_XML
 +        - $PY_MPI_UNIT_TEST_XML
 +        - $PY_ACCEPTANCE_TEST_XML
 +        - $PY_MPI_ACCEPTANCE_TEST_XML
 +    when: always
 +    expire_in: 1 week
 +  tags:
 +    - k8s-scilifelab
 +  # The dependency means we need to use the same tag restriction as upstream.
 +  needs:
- gmxapi-0.2:gcc-7:gmx2021:py-3.6.10:
++    - job: gromacs:gcc-10:build
 +      artifacts: true
 +
-     - .gmxapi-0.2:gcc-7:gmx2021
++gmxapi-0.2:gcc-10:gmx2021:py-3.6.10:
 +  extends:
- gmxapi-0.2:gcc-7:gmx2021:py-3.7.7:
++    - .gmxapi-0.2:gcc-10:gmx2021
 +    - .rules:merge-requests:release-2021
 +  variables:
 +    VENVPATH: "/root/venv/py3.6"
 +    PY_VER: "3.6.10"
 +
-     - .gmxapi-0.2:gcc-7:gmx2021
++gmxapi-0.2:gcc-10:gmx2021:py-3.7.7:
 +  extends:
- gmxapi-0.2:gcc-7:gmx2021:py-3.8.2:
++    - .gmxapi-0.2:gcc-10:gmx2021
 +    - .rules:merge-requests:release-2021
 +  variables:
 +    VENVPATH: "/root/venv/py3.7"
 +    PY_VER: "3.7.7"
 +
-     - .gmxapi-0.2:gcc-7:gmx2021
++gmxapi-0.2:gcc-10:gmx2021:py-3.8.2:
 +  extends:
++    - .gmxapi-0.2:gcc-10:gmx2021
 +    - .rules:merge-requests:release-2021
 +  variables:
 +    VENVPATH: "/root/venv/py3.8"
 +    PY_VER: "3.8.2"
index d88b74f4ceb3f18b82b3000e79aa8537d16bfd41,0000000000000000000000000000000000000000..53a8605d01c98e0ee122e48135ca227aac530a47
mode 100644,000000..100644
--- /dev/null
@@@ -1,53 -1,0 +1,53 @@@
- .gmxapi-0.3:gcc-7:gmx2022:
 +#
 +# Jobs to test gmxapi client (Python) packages
 +#
 +
-   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-gcc-7
++.gmxapi-0.3:gcc-10:gmx2022:
 +  extends:
 +    - .variables:default
 +    - .use-clang:base
-     - job: gromacs:gcc-7:build
++  image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-20.04-gcc-10
 +  stage: test
 +  variables:
 +    KUBERNETES_CPU_LIMIT: 2
 +    KUBERNETES_CPU_REQUEST: 2
 +    KUBERNETES_MEMORY_LIMIT: 2Gi
 +    KUBERNETES_MEMORY_REQUEST: 2Gi
 +    PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml
 +    PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml
 +    PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml
 +    PY_MPI_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-mpi-JUnitTestResults.xml
 +  script:
 +    - source $INSTALL_DIR/bin/GMXRC
 +    - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-py-gmxapi-0.3.sh
 +  artifacts:
 +    reports:
 +      junit:
 +        - $PY_UNIT_TEST_XML
 +        - $PY_MPI_UNIT_TEST_XML
 +        - $PY_ACCEPTANCE_TEST_XML
 +        - $PY_MPI_ACCEPTANCE_TEST_XML
 +    when: always
 +    expire_in: 1 week
 +  tags:
 +    - k8s-scilifelab
 +  # The dependency means we need to use the same tag restriction as upstream.
 +  needs:
- gmxapi-0.3:gcc-7:gmx2022:py-3.7.7:
++    - job: gromacs:gcc-10:build
 +      artifacts: true
 +
-     - .gmxapi-0.3:gcc-7:gmx2022
++gmxapi-0.3:gcc-10:gmx2022:py-3.7.7:
 +  extends:
- gmxapi-0.3:gcc-7:gmx2022:py-3.8.2:
++    - .gmxapi-0.3:gcc-10:gmx2022
 +    - .rules:merge-requests:master
 +  variables:
 +    VENVPATH: "/root/venv/py3.7"
 +    PY_VER: "3.7.7"
 +
-     - .gmxapi-0.3:gcc-7:gmx2022
++gmxapi-0.3:gcc-10:gmx2022:py-3.8.2:
 +  extends:
++    - .gmxapi-0.3:gcc-10:gmx2022
 +    - .rules:merge-requests:master
 +  variables:
 +    VENVPATH: "/root/venv/py3.8"
 +    PY_VER: "3.8.2"
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
index 5c62efc57d4cd660dc1293eab9e59117579c209e,6573583c7e50bb06fc1e0ee8594a4f66fc7bb7de..3c9b933758a5b4d5f77f0812f8cf01f867e69113
@@@ -431,10 -435,15 +435,15 @@@ static void nb_free_energy_kernel(cons
              }
              npair_within_cutoff++;
  
 -            if (rsq > rlistSquared)
++            if (rSq > rlistSquared)
+             {
+                 numExcludedPairsBeyondRlist++;
+             }
 -            if (rsq > 0)
 +            if (rSq > 0)
              {
                  /* Note that unlike in the nbnxn kernels, we do not need
 -                 * to clamp the value of rsq before taking the invsqrt
 +                 * to clamp the value of rSq before taking the invsqrt
                   * to avoid NaN in the LJ calculation, since here we do
                   * not calculate LJ interactions when C6 and C12 are zero.
                   */
                  }
              }
  
-             if (vdwInteractionTypeIsEwald && r < rVdw)
 -            if (vdwInteractionTypeIsEwald && (r < rvdw || !bPairIncluded))
++            if (vdwInteractionTypeIsEwald && (r < rVdw || !bPairIncluded))
              {
                  /* See comment in the preamble. When using LJ-Ewald interactions
                   * (unless we use a switch modifier) we subtract the reciprocal-space
       */
  #pragma omp atomic
      inc_nrnb(nrnb, eNR_NBKERNEL_FREE_ENERGY, nlist->nri * 12 + nlist->jindex[nri] * 150);
 -                  numExcludedPairsBeyondRlist, fr->rlist);
+     if (numExcludedPairsBeyondRlist > 0)
+     {
+         gmx_fatal(FARGS,
+                   "There are %d perturbed non-bonded pair interactions beyond the pair-list cutoff "
+                   "of %g nm, which is not supported. This can happen because the system is "
+                   "unstable or because intra-molecular interactions at long distances are "
+                   "excluded. If the "
+                   "latter is the case, you can try to increase nstlist or rlist to avoid this."
+                   "The error is likely triggered by the use of couple-intramol=no "
+                   "and the maximal distance in the decoupled molecule exceeding rlist.",
++                  numExcludedPairsBeyondRlist,
++                  fr->rlist);
+     }
  }
  
  typedef void (*KernelFunction)(const t_nblist* gmx_restrict nlist,
index c743b77c914b35b7b0e684d2e6a029e22dab6e29,8d7eb271086c8cf129c422602cb7d6263aecdb47..a0af9e6e7f2be0e60736dfcfbc4464295649403d
@@@ -771,14 -768,12 +772,14 @@@ void init_interaction_const_tables(FILE
  {
      if (EEL_PME_EWALD(ic->eeltype) || EVDW_PME(ic->vdwtype))
      {
 -        init_ewald_f_table(*ic, rlist, tableExtensionLength, ic->coulombEwaldTables.get(),
 -                           ic->vdwEwaldTables.get());
 +        init_ewald_f_table(
-                 *ic, tableExtensionLength, ic->coulombEwaldTables.get(), ic->vdwEwaldTables.get());
++                *ic, rlist, tableExtensionLength, ic->coulombEwaldTables.get(), ic->vdwEwaldTables.get());
          if (fp != nullptr)
          {
 -            fprintf(fp, "Initialized non-bonded Ewald tables, spacing: %.2e size: %zu\n\n",
 -                    1 / ic->coulombEwaldTables->scale, ic->coulombEwaldTables->tableF.size());
 +            fprintf(fp,
 +                    "Initialized non-bonded Ewald tables, spacing: %.2e size: %zu\n\n",
 +                    1 / ic->coulombEwaldTables->scale,
 +                    ic->coulombEwaldTables->tableF.size());
          }
      }
  }
Simple merge
Simple merge
index f660dfd7af9b6d278ab87758b250f5018403ceff,2450711c10f94220fa489b298d64329984bd982a..f0bb4100b963453842e8f680a31a0ec4c0232f00
@@@ -381,4 -388,32 +394,31 @@@ void ModularSimulator::checkInputForDis
      }
  }
  
 -    fr->step =
 -            int64_to_int(checkpointHeaderContents.step, "conversion of checkpoint to trajectory");
+ void ModularSimulator::readCheckpointToTrxFrame(t_trxframe*               fr,
+                                                 ReadCheckpointDataHolder* readCheckpointDataHolder,
+                                                 const CheckpointHeaderContents& checkpointHeaderContents)
+ {
+     GMX_RELEASE_ASSERT(checkpointHeaderContents.isModularSimulatorCheckpoint,
+                        "ModularSimulator::readCheckpointToTrxFrame can only read checkpoints "
+                        "written by modular simulator.");
+     fr->bStep = true;
++    fr->step = int64_to_int(checkpointHeaderContents.step, "conversion of checkpoint to trajectory");
+     fr->bTime = true;
+     fr->time  = checkpointHeaderContents.t;
+     fr->bAtoms = false;
+     StatePropagatorData::readCheckpointToTrxFrame(
+             fr, readCheckpointDataHolder->checkpointData(StatePropagatorData::checkpointID()));
+     if (readCheckpointDataHolder->keyExists(FreeEnergyPerturbationData::checkpointID()))
+     {
+         FreeEnergyPerturbationData::readCheckpointToTrxFrame(
+                 fr, readCheckpointDataHolder->checkpointData(FreeEnergyPerturbationData::checkpointID()));
+     }
+     else
+     {
+         FreeEnergyPerturbationData::readCheckpointToTrxFrame(fr, std::nullopt);
+     }
+ }
  } // namespace gmx
index 28e4ef9a250b26c6e1196fba1eaad3aa1286a443,92691193211a10c919416d3762627a8fda0a6003..866136a80959732268aff8236cb68bb270ceffc0
@@@ -489,38 -482,47 +490,55 @@@ constexpr auto c_currentVersion = Check
  } // namespace
  
  template<CheckpointDataOperation operation>
- void StatePropagatorData::Element::doCheckpointData(CheckpointData<operation>* checkpointData,
-                                                     const t_commrec*           cr)
+ void StatePropagatorData::doCheckpointData(CheckpointData<operation>* checkpointData)
+ {
+     checkpointVersion(checkpointData, "StatePropagatorData version", c_currentVersion);
+     checkpointData->scalar("numAtoms", &totalNumAtoms_);
+     if (operation == CheckpointDataOperation::Read)
+     {
+         xGlobal_.resizeWithPadding(totalNumAtoms_);
+         vGlobal_.resizeWithPadding(totalNumAtoms_);
+     }
+     checkpointData->arrayRef("positions", makeCheckpointArrayRef<operation>(xGlobal_));
+     checkpointData->arrayRef("velocities", makeCheckpointArrayRef<operation>(vGlobal_));
+     checkpointData->tensor("box", box_);
+     checkpointData->scalar("ddpCount", &ddpCount_);
+     checkpointData->scalar("ddpCountCgGl", &ddpCountCgGl_);
+     checkpointData->arrayRef("cgGl", makeCheckpointArrayRef<operation>(cgGl_));
+ }
+ void StatePropagatorData::Element::saveCheckpointState(std::optional<WriteCheckpointData> checkpointData,
+                                                        const t_commrec*                   cr)
  {
-     ArrayRef<RVec> xGlobalRef;
-     ArrayRef<RVec> vGlobalRef;
      if (DOMAINDECOMP(cr))
      {
-         if (MASTER(cr))
-         {
-             xGlobalRef = statePropagatorData_->xGlobal_;
-             vGlobalRef = statePropagatorData_->vGlobal_;
-         }
-         if (operation == CheckpointDataOperation::Write)
-         {
-             dd_collect_vec(cr->dd,
-                            statePropagatorData_->ddpCount_,
-                            statePropagatorData_->ddpCountCgGl_,
-                            statePropagatorData_->cgGl_,
-                            statePropagatorData_->x_,
-                            xGlobalRef);
-             dd_collect_vec(cr->dd,
-                            statePropagatorData_->ddpCount_,
-                            statePropagatorData_->ddpCountCgGl_,
-                            statePropagatorData_->cgGl_,
-                            statePropagatorData_->v_,
-                            vGlobalRef);
-         }
+         // Collect state from all ranks into global vectors
 -        dd_collect_vec(cr->dd, statePropagatorData_->ddpCount_, statePropagatorData_->ddpCountCgGl_,
 -                       statePropagatorData_->cgGl_, statePropagatorData_->x_,
++        dd_collect_vec(cr->dd,
++                       statePropagatorData_->ddpCount_,
++                       statePropagatorData_->ddpCountCgGl_,
++                       statePropagatorData_->cgGl_,
++                       statePropagatorData_->x_,
+                        statePropagatorData_->xGlobal_);
 -        dd_collect_vec(cr->dd, statePropagatorData_->ddpCount_, statePropagatorData_->ddpCountCgGl_,
 -                       statePropagatorData_->cgGl_, statePropagatorData_->v_,
++        dd_collect_vec(cr->dd,
++                       statePropagatorData_->ddpCount_,
++                       statePropagatorData_->ddpCountCgGl_,
++                       statePropagatorData_->cgGl_,
++                       statePropagatorData_->v_,
+                        statePropagatorData_->vGlobal_);
      }
      else
      {
-         xGlobalRef = statePropagatorData_->x_;
-         vGlobalRef = statePropagatorData_->v_;
+         // Everything is local - copy local vectors into global ones
+         statePropagatorData_->xGlobal_.resizeWithPadding(statePropagatorData_->totalNumAtoms());
+         statePropagatorData_->vGlobal_.resizeWithPadding(statePropagatorData_->totalNumAtoms());
 -        std::copy(statePropagatorData_->x_.begin(), statePropagatorData_->x_.end(),
++        std::copy(statePropagatorData_->x_.begin(),
++                  statePropagatorData_->x_.end(),
+                   statePropagatorData_->xGlobal_.begin());
 -        std::copy(statePropagatorData_->v_.begin(), statePropagatorData_->v_.end(),
++        std::copy(statePropagatorData_->v_.begin(),
++                  statePropagatorData_->v_.end(),
+                   statePropagatorData_->vGlobal_.begin());
      }
      if (MASTER(cr))
      {
@@@ -573,14 -563,21 +579,26 @@@ void StatePropagatorData::Element::rest
      // Copy data to global state to be distributed by DD at setup stage
      if (DOMAINDECOMP(cr) && MASTER(cr))
      {
 -        updateGlobalState(statePropagatorData_->globalState_, statePropagatorData_->xGlobal_,
 -                          statePropagatorData_->vGlobal_, statePropagatorData_->box_,
 -                          statePropagatorData_->ddpCount_, statePropagatorData_->ddpCountCgGl_,
 +        updateGlobalState(statePropagatorData_->globalState_,
 +                          statePropagatorData_->xGlobal_,
 +                          statePropagatorData_->vGlobal_,
 +                          statePropagatorData_->box_,
 +                          statePropagatorData_->ddpCount_,
 +                          statePropagatorData_->ddpCountCgGl_,
                            statePropagatorData_->cgGl_);
      }
 -        std::copy(statePropagatorData_->xGlobal_.begin(), statePropagatorData_->xGlobal_.end(),
+     // Everything is local - copy global vectors to local ones
+     if (!DOMAINDECOMP(cr))
+     {
+         statePropagatorData_->x_.resizeWithPadding(statePropagatorData_->totalNumAtoms_);
+         statePropagatorData_->v_.resizeWithPadding(statePropagatorData_->totalNumAtoms_);
 -        std::copy(statePropagatorData_->vGlobal_.begin(), statePropagatorData_->vGlobal_.end(),
++        std::copy(statePropagatorData_->xGlobal_.begin(),
++                  statePropagatorData_->xGlobal_.end(),
+                   statePropagatorData_->x_.begin());
++        std::copy(statePropagatorData_->vGlobal_.begin(),
++                  statePropagatorData_->vGlobal_.end(),
+                   statePropagatorData_->v_.begin());
+     }
  }
  
  const std::string& StatePropagatorData::Element::clientID()
index 542d642f498759a6a1e5f9961913b45609c4a3b6,c650ad276e34aa9b16ddb8779c009894cd7630b7..2e5e15850d7aa8c2c8f2ca1eccefa4ddbfc4246e
@@@ -388,13 -390,20 +388,21 @@@ static inline SimdDouble gmx_simdcall f
      iExponent = svsub_s64_x(
              pg, svreinterpret_s64_u64(svlsr_n_u64_x(pg, svreinterpret_u64_s64(iExponent), 52)), exponentBias);
  
-     exponent->simdInternal_ = iExponent;
  
-     return { svreinterpret_f64_s64(
 -    svfloat64_t result = svreinterpret_f64_s64(svorr_s64_x(
 -            pg, svand_s64_x(pg, svreinterpret_s64_f64(value.simdInternal_), mantissaMask),
 -            svreinterpret_s64_f64(half)));
++    svfloat64_t result = svreinterpret_f64_s64(
 +            svorr_s64_x(pg,
 +                        svand_s64_x(pg, svreinterpret_s64_f64(value.simdInternal_), mantissaMask),
-                         svreinterpret_s64_f64(half))) };
++                        svreinterpret_s64_f64(half)));
+     if (opt == MathOptimization::Safe)
+     {
+         svbool_t valueIsZero = svcmpeq_n_f64(pg, value.simdInternal_, 0.0);
+         iExponent            = svsel_s64(valueIsZero, svdup_s64(0), iExponent);
+         result               = svsel_f64(valueIsZero, value.simdInternal_, result);
+     }
+     exponent->simdInternal_ = iExponent;
+     return { result };
  }
  
  template<MathOptimization opt = MathOptimization::Safe>
index a7f2737c27c5f65e9428f139c49989311fd3bce8,1171124f1924536ee7887042bc84d506f5dac37e..2945ea66a03df0b95445dfc3bdd2ad37a8da11a5
@@@ -392,12 -392,21 +392,21 @@@ static inline SimdFloat gmx_simdcall fr
      iExponent = svand_s32_x(pg, svreinterpret_s32_f32(value.simdInternal_), exponentMask);
      iExponent = svsub_s32_x(
              pg, svreinterpret_s32_u32(svlsr_n_u32_x(pg, svreinterpret_u32_s32(iExponent), 23)), exponentBias);
-     exponent->simdInternal_ = iExponent;
  
-     return { svreinterpret_f32_s32(
 -
 -    svfloat32_t result = svreinterpret_f32_s32(svorr_s32_x(
 -            pg, svand_s32_x(pg, svreinterpret_s32_f32(value.simdInternal_), mantissaMask),
 -            svreinterpret_s32_f32(half)));
++    svfloat32_t result = svreinterpret_f32_s32(
 +            svorr_s32_x(pg,
 +                        svand_s32_x(pg, svreinterpret_s32_f32(value.simdInternal_), mantissaMask),
-                         svreinterpret_s32_f32(half))) };
++                        svreinterpret_s32_f32(half)));
+     if (opt == MathOptimization::Safe)
+     {
+         svbool_t valueIsZero = svcmpeq_n_f32(pg, value.simdInternal_, 0.0F);
+         iExponent            = svsel_s32(valueIsZero, svdup_s32(0), iExponent);
+         result               = svsel_f32(valueIsZero, value.simdInternal_, result);
+     }
+     exponent->simdInternal_ = iExponent;
+     return { result };
  }
  
  template<MathOptimization opt = MathOptimization::Safe>
index ab7a55ea54fa761b89338628c86affd7166197ac,adc754772e1e19dc39f2dd9948f623d9e8b49c4c..24fa2b3fd4845e5323f91a667bfc352da10c5ac0
@@@ -113,6 -113,21 +113,22 @@@ void dumpKeyValueTree(TextWriter* write
              dumpKeyValueTree(writer, value.asObject());
              writer->wrapperSettings().setIndent(oldIndent);
          }
 -                 && std::all_of(value.asArray().values().begin(), value.asArray().values().end(),
+         else if (value.isArray()
++                 && std::all_of(value.asArray().values().begin(),
++                                value.asArray().values().end(),
+                                 [](const auto& elem) { return elem.isObject(); }))
+         {
+             // Array containing only objects
+             writer->writeString(prop.key());
+             writer->writeLine(":");
+             int oldIndent = writer->wrapperSettings().indent();
+             writer->wrapperSettings().setIndent(oldIndent + 2);
+             for (const auto& elem : value.asArray().values())
+             {
+                 dumpKeyValueTree(writer, elem.asObject());
+             }
+             writer->wrapperSettings().setIndent(oldIndent);
+         }
          else
          {
              int indent = writer->wrapperSettings().indent();
index 56a21bd394fa2b85aab7faa9a50b08c1acfebe90,860729a26afe1fee2ff686c61f62d490e33df611..63c595a630cf9dc0701ccd358694f6c64de9f485
@@@ -51,23 -47,20 +51,24 @@@ if(GMX_FAHCORE
      # may break some generators, according to CMake documentation. If
      # so, we can consider adding some dummy file to make it work.
      add_library(fahcore $<TARGET_OBJECTS:mdrun_objlib>)
 -    target_link_libraries(fahcore PRIVATE ${GMX_COMMON_LIBRARIES})
 +    target_link_libraries(fahcore PRIVATE ${GMX_COMMON_LIBRARIES} legacy_api)
  elseif(GMX_BUILD_MDRUN_ONLY)
 -    add_executable(mdrun $<TARGET_OBJECTS:mdrun_objlib> mdrun_main.cpp)
 -    gmx_target_compile_options(mdrun)
 -    target_include_directories(mdrun SYSTEM BEFORE PRIVATE ${PROJECT_SOURCE_DIR}/src/external/thread_mpi/include)
 -    target_compile_definitions(mdrun PRIVATE HAVE_CONFIG_H)
 -    target_link_libraries(mdrun libgromacs
 -        ${GMX_COMMON_LIBRARIES}
 -        ${GMX_EXE_LINKER_FLAGS})
+     message(STATUS "The mdrun-only build is deprecated")
 +    add_executable(mdrun-only $<TARGET_OBJECTS:mdrun_objlib> mdrun_main.cpp)
 +    gmx_target_compile_options(mdrun-only)
 +    target_include_directories(mdrun-only SYSTEM BEFORE PRIVATE ${PROJECT_SOURCE_DIR}/src/external/thread_mpi/include)
 +    target_compile_definitions(mdrun-only PRIVATE HAVE_CONFIG_H)
 +    target_link_libraries(mdrun-only PRIVATE
 +                          common
 +                          legacy_modules
 +                          libgromacs
 +                          ${GMX_COMMON_LIBRARIES}
 +                          ${GMX_EXE_LINKER_FLAGS}
 +                          )
      set(BINARY_NAME "mdrun${GMX_BINARY_SUFFIX}")
 -    set_target_properties(mdrun PROPERTIES
 +    set_target_properties(mdrun-only PROPERTIES
          OUTPUT_NAME "${BINARY_NAME}")
 -    install(TARGETS mdrun DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT mdrun)
 +    install(TARGETS mdrun-only DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT mdrun-only)
      file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/gmx-completion-${BINARY_NAME}.bash
           "complete -o nospace -F _gmx_mdrun_compl ${BINARY_NAME}")
      install(FILES ${CMAKE_CURRENT_BINARY_DIR}/gmx-completion-${BINARY_NAME}.bash
index 4a7774237f16fb8d09fbae753fc90c7beb4b1dd9,e4eccebd696ee00f00d79b2dad0ac391dda809ab..357882bb4843d9ed2b35854e8cdc0c725cbf6ff1
@@@ -231,6 -230,18 +231,20 @@@ gmx_add_gtest_executable(${exename
          $<TARGET_OBJECTS:mdrun_objlib>
  )
  target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
 +# TODO: Link specific modules: topology
 +target_link_libraries(${exename} PRIVATE legacy_modules)
  gmx_register_gtest_test(${testname} ${exename} OPENMP_THREADS 2 INTEGRATION_TEST IGNORE_LEAKS)
+ # End-to-end tests comparing different simulator code paths
+ set(testname "MdrunSimulatorComparison")
+ set(exename "mdrun-simulator-comparison-test")
+ gmx_add_gtest_executable(${exename}
+         CPP_SOURCE_FILES
+         # files with code for tests
+         simulator.cpp
+         # pseudo-library for code for mdrun
+         $<TARGET_OBJECTS:mdrun_objlib>
+         )
+ target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure)
+ gmx_register_gtest_test(${testname} ${exename} OPENMP_THREADS 2 INTEGRATION_TEST IGNORE_LEAKS)
index 0000000000000000000000000000000000000000,dcee1703feef6f9d78db9d1e2905ee771616e938..2ddd809420e552b34780ca2a945ffc6a38f1dd03
mode 000000,100644..100644
--- /dev/null
@@@ -1,0 -1,171 +1,174 @@@
 -    const TrajectoryTolerances trajectoryTolerances{ defaultRealTolerance(), defaultRealTolerance(),
 -                                                     defaultRealTolerance(), defaultRealTolerance() };
+ /*
+  * This file is part of the GROMACS molecular simulation package.
+  *
+  * Copyright (c) 2020, by the GROMACS development team, led by
+  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+  * and including many others, as listed in the AUTHORS file in the
+  * top-level source directory and at http://www.gromacs.org.
+  *
+  * GROMACS is free software; you can redistribute it and/or
+  * modify it under the terms of the GNU Lesser General Public License
+  * as published by the Free Software Foundation; either version 2.1
+  * of the License, or (at your option) any later version.
+  *
+  * GROMACS is distributed in the hope that it will be useful,
+  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  * Lesser General Public License for more details.
+  *
+  * You should have received a copy of the GNU Lesser General Public
+  * License along with GROMACS; if not, see
+  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+  *
+  * If you want to redistribute modifications to GROMACS, please
+  * consider that scientific software is very special. Version
+  * control is crucial - bugs must be traceable. We will be happy to
+  * consider code for inclusion in the official distribution, but
+  * derived work must not be called official GROMACS. Details are found
+  * in the README & COPYING files - if they are missing, get the
+  * official version at http://www.gromacs.org.
+  *
+  * To help us fund GROMACS development, we humbly ask that you cite
+  * the research papers on the package. Check out http://www.gromacs.org.
+  */
+ /*! \internal \file
+  * \brief Tests for checkpoint writing sanity checks
+  *
+  * Checks that final checkpoint is equal to final trajectory output.
+  *
+  * \author Pascal Merz <pascal.merz@me.com>
+  * \ingroup module_mdrun_integration_tests
+  */
+ #include "gmxpre.h"
+ #include "config.h"
+ #include "gromacs/utility/strconvert.h"
+ #include "gromacs/utility/stringutil.h"
+ #include "testutils/simulationdatabase.h"
+ #include "moduletest.h"
+ #include "simulatorcomparison.h"
+ #include "trajectoryreader.h"
+ namespace gmx::test
+ {
+ namespace
+ {
+ class CheckpointCoordinatesSanityChecks :
+     public MdrunTestFixture,
+     public ::testing::WithParamInterface<std::tuple<std::string, std::string, std::string, std::string>>
+ {
+ public:
+     void runSimulation(MdpFieldValues     mdpFieldValues,
+                        int                numSteps,
+                        const std::string& trrFileName,
+                        const std::string& cptFileName)
+     {
+         mdpFieldValues["nsteps"] = toString(numSteps);
+         // Trajectories have the initial and the last frame
+         mdpFieldValues["nstxout"] = toString(numSteps);
+         mdpFieldValues["nstvout"] = toString(numSteps);
+         mdpFieldValues["nstfout"] = toString(0);
+         // Run grompp
+         runner_.useStringAsMdpFile(prepareMdpFileContents(mdpFieldValues));
+         runGrompp(&runner_);
+         // Do first mdrun
+         runner_.fullPrecisionTrajectoryFileName_ = trrFileName;
+         runMdrun(&runner_, { { "-cpo", cptFileName } });
+     }
+     static void compareCptAndTrr(const std::string&          trrFileName,
+                                  const std::string&          cptFileName,
+                                  const TrajectoryComparison& trajectoryComparison)
+     {
+         TrajectoryFrameReader trrReader(trrFileName);
+         TrajectoryFrameReader cptReader(cptFileName);
+         // Checkpoint has at least one frame
+         EXPECT_TRUE(cptReader.readNextFrame());
+         // Trajectory has at least two frames
+         EXPECT_TRUE(trrReader.readNextFrame());
+         EXPECT_NO_THROW(trrReader.frame());
+         EXPECT_TRUE(trrReader.readNextFrame());
+         // Now compare frames
+         trajectoryComparison(cptReader.frame(), trrReader.frame());
+         // Files had exactly 1 / 2 frames
+         EXPECT_FALSE(cptReader.readNextFrame());
+         EXPECT_FALSE(trrReader.readNextFrame());
+     }
+ };
+ TEST_P(CheckpointCoordinatesSanityChecks, WithinTolerances)
+ {
+     const auto& params              = GetParam();
+     const auto& simulationName      = std::get<0>(params);
+     const auto& integrator          = std::get<1>(params);
+     const auto& temperatureCoupling = std::get<2>(params);
+     const auto& pressureCoupling    = std::get<3>(params);
+     // Specify how trajectory frame matching must work.
+     TrajectoryFrameMatchSettings trajectoryMatchSettings{ true,
+                                                           true,
+                                                           true,
+                                                           ComparisonConditions::MustCompare,
+                                                           ComparisonConditions::MustCompare,
+                                                           ComparisonConditions::NoComparison,
+                                                           MaxNumFrames::compareAllFrames() };
+     if (integrator == "md-vv")
+     {
+         // When using md-vv and modular simulator, the velocities are expected to be off by
+         // 1/2 dt between checkpoint (top of the loop) and trajectory (full time step state)
+         trajectoryMatchSettings.velocitiesComparison = ComparisonConditions::NoComparison;
+     }
 -            simulationName.c_str(), integrator.c_str(), temperatureCoupling.c_str(),
++    const TrajectoryTolerances trajectoryTolerances{
++        defaultRealTolerance(), defaultRealTolerance(), defaultRealTolerance(), defaultRealTolerance()
++    };
+     const auto mdpFieldValues =
+             prepareMdpFieldValues(simulationName, integrator, temperatureCoupling, pressureCoupling);
+     runner_.useTopGroAndNdxFromDatabase(simulationName);
+     // Set file names
+     const auto cptFileName = fileManager_.getTemporaryFilePath(".cpt");
+     const auto trrFileName = fileManager_.getTemporaryFilePath(".trr");
+     SCOPED_TRACE(formatString(
+             "Checking the sanity of the checkpointed coordinates using system '%s' "
+             "with integrator '%s', '%s' temperature coupling, and '%s' pressure coupling ",
++            simulationName.c_str(),
++            integrator.c_str(),
++            temperatureCoupling.c_str(),
+             pressureCoupling.c_str()));
+     SCOPED_TRACE("End of trajectory sanity");
+     // Running a few steps - we expect the checkpoint to be equal
+     // to the final configuration
+     runSimulation(mdpFieldValues, 16, trrFileName, cptFileName);
+     compareCptAndTrr(trrFileName, cptFileName, { trajectoryMatchSettings, trajectoryTolerances });
+ }
+ #if !GMX_GPU_OPENCL
+ INSTANTIATE_TEST_CASE_P(CheckpointCoordinatesAreSane,
+                         CheckpointCoordinatesSanityChecks,
+                         ::testing::Combine(::testing::Values("spc2"),
+                                            ::testing::Values("md", "md-vv"),
+                                            ::testing::Values("no"),
+                                            ::testing::Values("no")));
+ #else
+ INSTANTIATE_TEST_CASE_P(DISABLED_CheckpointCoordinatesAreSane,
+                         CheckpointCoordinatesSanityChecks,
+                         ::testing::Combine(::testing::Values("spc2"),
+                                            ::testing::Values("md", "md-vv"),
+                                            ::testing::Values("no"),
+                                            ::testing::Values("no")));
+ #endif
+ } // namespace
+ } // namespace gmx::test
index 6c39160bf3fb22b79c74e827069e69776a2a10ec,3159e3db4ed0f877572d2c41e6c98bff198dc7c3..9072e7b98ec93a4f53564a93a08fb383bbfa5c13
@@@ -95,8 -95,18 +95,19 @@@ TEST_P(FreeEnergyReferenceTest, WithinT
      const auto  maxNumWarnings   = std::get<1>(GetParam());
      const auto& interactionsList = std::get<2>(GetParam());
  
-     // TODO In similar tests, we are checking if the tests
-     //      can be run with the number of MPI ranks available
+     // As these tests check reproducibility, we restrict the maximum number
+     // of ranks to allow us to keep the tolerances tight. See also #3741.
+     const int     numRanksAvailable = getNumberOfTestMpiRanks();
+     constexpr int maxNumRanks       = 8;
+     if (numRanksAvailable > maxNumRanks)
+     {
+         fprintf(stdout,
+                 "The FEP tests cannot run with %d ranks.\n"
+                 "The maximum number of ranks supported is %d.",
 -                numRanksAvailable, maxNumRanks);
++                numRanksAvailable,
++                maxNumRanks);
+         return;
+     }
  
      SCOPED_TRACE(formatString("Comparing FEP simulation '%s' to reference", simulationName.c_str()));
  
      // Check that the energies agree with the refdata within tolerance.
      checkEnergiesAgainstReferenceData(simulationEdrFileName, energyTermsToCompare, &rootChecker);
      // Check that the trajectories agree with the refdata within tolerance.
-     if (testAllTrajectoryFrames)
+     if (testTwoTrajectoryFrames)
      {
-         checkTrajectoryAgainstReferenceData(simulationTrajectoryFileName, trajectoryComparison, &rootChecker);
 -        checkTrajectoryAgainstReferenceData(simulationTrajectoryFileName, trajectoryComparison,
 -                                            &rootChecker, MaxNumFrames(2));
++        checkTrajectoryAgainstReferenceData(
++                simulationTrajectoryFileName, trajectoryComparison, &rootChecker, MaxNumFrames(2));
      }
-     else
+     else if (testOneTrajectoryFrame)
      {
 -        checkTrajectoryAgainstReferenceData(simulationTrajectoryFileName, trajectoryComparison,
 -                                            &rootChecker, MaxNumFrames(1));
 +        checkTrajectoryAgainstReferenceData(
 +                simulationTrajectoryFileName, trajectoryComparison, &rootChecker, MaxNumFrames(1));
      }
 -        checkTrajectoryAgainstReferenceData(simulationTrajectoryFileName, trajectoryComparison,
 -                                            &rootChecker, MaxNumFrames(0));
+     else
+     {
++        checkTrajectoryAgainstReferenceData(
++                simulationTrajectoryFileName, trajectoryComparison, &rootChecker, MaxNumFrames(0));
+     }
      if (File::exists(simulationDhdlFileName, File::returnFalseOnError))
      {
          TextInputFile dhdlFile(simulationDhdlFileName);
index b8aec654a78a12053311487a1d886cd8c35283ca,693f22c7731e59f85ee238261a1e39729e51e3c3..65418e8c899d2245f58ac6c5bf867691e1d606c5
@@@ -116,18 -123,15 +124,18 @@@ TEST_P(SimulatorComparisonTest, WithinT
              "Comparing two simulations of '%s' "
              "with integrator '%s', '%s' temperature coupling, and '%s' pressure coupling "
              "switching environment variable '%s'",
 -            simulationName.c_str(), integrator.c_str(), tcoupling.c_str(), pcoupling.c_str(),
 +            simulationName.c_str(),
 +            integrator.c_str(),
 +            tcoupling.c_str(),
 +            pcoupling.c_str(),
              environmentVariable.c_str()));
  
 -    const auto mdpFieldValues = prepareMdpFieldValues(simulationName.c_str(), integrator.c_str(),
 -                                                      tcoupling.c_str(), pcoupling.c_str());
 +    const auto mdpFieldValues = prepareMdpFieldValues(
 +            simulationName.c_str(), integrator.c_str(), tcoupling.c_str(), pcoupling.c_str());
  
      EnergyTermsToCompare energyTermsToCompare{ {
-             { interaction_function[F_EPOT].longname, relativeToleranceAsPrecisionDependentUlp(10.0, 100, 80) },
-             { interaction_function[F_EKIN].longname, relativeToleranceAsPrecisionDependentUlp(60.0, 100, 80) },
+             { interaction_function[F_EPOT].longname, relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
+             { interaction_function[F_EKIN].longname, relativeToleranceAsPrecisionDependentUlp(60.0, 200, 160) },
              { interaction_function[F_PRES].longname,
                relativeToleranceAsPrecisionDependentFloatingPoint(10.0, 0.01, 0.001) },
      } };