Merge branch release-2021 into merge-2021-into-master
[alexxy/gromacs.git] / src / gromacs / mdrun / rerun.cpp
index 36333d3c94273c5ecc147dfce8c94fc1d6824988..78271fae4824e620cf3969fc14ce6097ad9a40e2 100644 (file)
@@ -145,13 +145,11 @@ using gmx::VirtualSitesHandler;
  * \param[in,out] globalState     The global state container
  * \param[in]     constructVsites When true, vsite coordinates are constructed
  * \param[in]     vsite           Vsite setup, can be nullptr when \p constructVsites = false
- * \param[in]     timeStep        Time step, used for constructing vsites
  */
 static void prepareRerunState(const t_trxframe&          rerunFrame,
                               t_state*                   globalState,
                               bool                       constructVsites,
-                              const VirtualSitesHandler* vsite,
-                              double                     timeStep)
+                              const VirtualSitesHandler* vsite)
 {
     auto x      = makeArrayRef(globalState->x);
     auto rerunX = arrayRefFromArray(reinterpret_cast<gmx::RVec*>(rerunFrame.x), globalState->natoms);
@@ -162,7 +160,7 @@ static void prepareRerunState(const t_trxframe&          rerunFrame,
     {
         GMX_ASSERT(vsite, "Need valid vsite for constructing vsites");
 
-        vsite->construct(globalState->x, timeStep, globalState->v, globalState->box);
+        vsite->construct(globalState->x, globalState->v, globalState->box, gmx::VSiteOperation::PositionsAndVelocities);
     }
 }
 
@@ -174,7 +172,7 @@ void gmx::LegacySimulator::do_rerun()
     // alias to avoid a large ripple of nearly useless changes.
     // t_inputrec is being replaced by IMdpOptionsProvider, so this
     // will go away eventually.
-    t_inputrec*       ir = inputrec;
+    const t_inputrec* ir = inputrec;
     int64_t           step, step_rel;
     double            t;
     bool              isLastStep               = false;
@@ -184,22 +182,13 @@ void gmx::LegacySimulator::do_rerun()
     t_trxstatus*      status = nullptr;
     rvec              mu_tot;
     t_trxframe        rerun_fr;
-    gmx_localtop_t    top(top_global->ffparams);
+    gmx_localtop_t    top(top_global.ffparams);
     ForceBuffers      f;
     gmx_global_stat_t gstat;
     gmx_shellfc_t*    shellfc;
 
     double cycles;
 
-    /* Domain decomposition could incorrectly miss a bonded
-       interaction, but checking for that requires a global
-       communication stage, which does not otherwise happen in DD
-       code. So we do that alongside the first global energy reduction
-       after a new DD is made. These variables handle whether the
-       check happens, and the result it returns. */
-    bool shouldCheckNumberOfBondedInteractions = false;
-    int  totalNumberOfBondedInteractions       = -1;
-
     SimulationSignals signals;
     // Most global communnication stages don't propagate mdrun
     // signals, and will use this object to achieve that.
@@ -212,7 +201,7 @@ void gmx::LegacySimulator::do_rerun()
                     "be available in a different form in a future version of GROMACS, "
                     "e.g. gmx rerun -f.");
 
-    if (ir->efep != efepNO
+    if (ir->efep != FreeEnergyPerturbationType::No
         && (mdAtoms->mdatoms()->nMassPerturbed > 0 || (constr && constr->havePerturbedConstraints())))
     {
         gmx_fatal(FARGS,
@@ -248,8 +237,9 @@ void gmx::LegacySimulator::do_rerun()
     {
         gmx_fatal(FARGS, "Multiple simulations not supported by rerun.");
     }
-    if (std::any_of(ir->opts.annealing, ir->opts.annealing + ir->opts.ngtc,
-                    [](int i) { return i != eannNO; }))
+    if (std::any_of(ir->opts.annealing, ir->opts.annealing + ir->opts.ngtc, [](SimulatedAnnealing i) {
+            return i != SimulatedAnnealing::No;
+        }))
     {
         gmx_fatal(FARGS, "Simulated annealing not supported by rerun.");
     }
@@ -268,38 +258,70 @@ void gmx::LegacySimulator::do_rerun()
     }
 
     /* Settings for rerun */
-    ir->nstlist              = 1;
-    ir->nstcalcenergy        = 1;
+    {
+        // TODO: Avoid changing inputrec (#3854)
+        auto* nonConstInputrec               = const_cast<t_inputrec*>(inputrec);
+        nonConstInputrec->nstlist            = 1;
+        nonConstInputrec->nstcalcenergy      = 1;
+        nonConstInputrec->nstxout_compressed = 0;
+    }
     int        nstglobalcomm = 1;
     const bool bNS           = true;
 
-    ir->nstxout_compressed         = 0;
-    const SimulationGroups* groups = &top_global->groups;
-    if (ir->eI == eiMimic)
+    const SimulationGroups* groups = &top_global.groups;
+    if (ir->eI == IntegrationAlgorithm::Mimic)
     {
-        auto nonConstGlobalTopology                          = const_cast<gmx_mtop_t*>(top_global);
-        nonConstGlobalTopology->intermolecularExclusionGroup = genQmmmIndices(*top_global);
+        auto* nonConstGlobalTopology                         = const_cast<gmx_mtop_t*>(&top_global);
+        nonConstGlobalTopology->intermolecularExclusionGroup = genQmmmIndices(top_global);
     }
     int*                fep_state = MASTER(cr) ? &state_global->fep_state : nullptr;
     gmx::ArrayRef<real> lambda    = MASTER(cr) ? state_global->lambda : gmx::ArrayRef<real>();
-    initialize_lambdas(fplog, *ir, MASTER(cr), fep_state, lambda);
+    initialize_lambdas(fplog,
+                       ir->efep,
+                       ir->bSimTemp,
+                       *ir->fepvals,
+                       ir->simtempvals->temperatures,
+                       gmx::arrayRefFromArray(ir->opts.ref_t, ir->opts.ngtc),
+                       MASTER(cr),
+                       fep_state,
+                       lambda);
     const bool        simulationsShareState = false;
-    gmx_mdoutf*       outf = init_mdoutf(fplog, nfile, fnm, mdrunOptions, cr, outputProvider,
-                                   mdModulesNotifier, ir, top_global, oenv, wcycle,
-                                   StartingBehavior::NewSimulation, simulationsShareState, ms);
-    gmx::EnergyOutput energyOutput(mdoutf_get_fp_ene(outf), top_global, ir, pull_work,
-                                   mdoutf_get_fp_dhdl(outf), true, StartingBehavior::NewSimulation,
-                                   simulationsShareState, mdModulesNotifier);
+    gmx_mdoutf*       outf                  = init_mdoutf(fplog,
+                                   nfile,
+                                   fnm,
+                                   mdrunOptions,
+                                   cr,
+                                   outputProvider,
+                                   mdModulesNotifiers,
+                                   ir,
+                                   top_global,
+                                   oenv,
+                                   wcycle,
+                                   StartingBehavior::NewSimulation,
+                                   simulationsShareState,
+                                   ms);
+    gmx::EnergyOutput energyOutput(mdoutf_get_fp_ene(outf),
+                                   top_global,
+                                   *ir,
+                                   pull_work,
+                                   mdoutf_get_fp_dhdl(outf),
+                                   true,
+                                   StartingBehavior::NewSimulation,
+                                   simulationsShareState,
+                                   mdModulesNotifiers);
 
     gstat = global_stat_init(ir);
 
     /* Check for polarizable models and flexible constraints */
-    shellfc = init_shell_flexcon(fplog, top_global, constr ? constr->numFlexibleConstraints() : 0,
-                                 ir->nstcalcenergy, DOMAINDECOMP(cr),
+    shellfc = init_shell_flexcon(fplog,
+                                 top_global,
+                                 constr ? constr->numFlexibleConstraints() : 0,
+                                 ir->nstcalcenergy,
+                                 DOMAINDECOMP(cr),
                                  runScheduleWork->simulationWork.useGpuPme);
 
     {
-        double io = compute_io(ir, top_global->natoms, *groups, energyOutput.numEnergyTerms(), 1);
+        double io = compute_io(ir, top_global.natoms, *groups, energyOutput.numEnergyTerms(), 1);
         if ((io > 2000) && MASTER(cr))
         {
             fprintf(stderr, "\nWARNING: This run will generate roughly %.0f Mb of data\n\n", io);
@@ -314,13 +336,30 @@ void gmx::LegacySimulator::do_rerun()
     {
         stateInstance = std::make_unique<t_state>();
         state         = stateInstance.get();
-        dd_init_local_state(cr->dd, state_global, state);
+        dd_init_local_state(*cr->dd, state_global, state);
 
         /* Distribute the charge groups over the nodes from the master node */
-        dd_partition_system(fplog, mdlog, ir->init_step, cr, TRUE, 1, state_global, *top_global, ir,
-                            imdSession, pull_work, state, &f, mdAtoms, &top, fr, vsite, constr,
-                            nrnb, nullptr, FALSE);
-        shouldCheckNumberOfBondedInteractions = true;
+        dd_partition_system(fplog,
+                            mdlog,
+                            ir->init_step,
+                            cr,
+                            TRUE,
+                            1,
+                            state_global,
+                            top_global,
+                            *ir,
+                            imdSession,
+                            pull_work,
+                            state,
+                            &f,
+                            mdAtoms,
+                            &top,
+                            fr,
+                            vsite,
+                            constr,
+                            nrnb,
+                            nullptr,
+                            FALSE);
     }
     else
     {
@@ -328,44 +367,67 @@ void gmx::LegacySimulator::do_rerun()
         /* Copy the pointer to the global state */
         state = state_global;
 
-        mdAlgorithmsSetupAtomData(cr, ir, *top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc);
+        mdAlgorithmsSetupAtomData(cr, *ir, top_global, &top, fr, &f, mdAtoms, constr, vsite, shellfc);
     }
 
-    auto mdatoms = mdAtoms->mdatoms();
+    auto* mdatoms = mdAtoms->mdatoms();
 
     // NOTE: The global state is no longer used at this point.
     // But state_global is still used as temporary storage space for writing
     // the global state to file and potentially for replica exchange.
     // (Global topology should persist.)
 
-    update_mdatoms(mdatoms, state->lambda[efptMASS]);
+    update_mdatoms(mdatoms, state->lambda[FreeEnergyPerturbationCouplingType::Mass]);
 
-    if (ir->efep != efepNO && ir->fepvals->nstdhdl != 0)
+    if (ir->efep != FreeEnergyPerturbationType::No && ir->fepvals->nstdhdl != 0)
     {
         doFreeEnergyPerturbation = true;
     }
 
     {
-        int cglo_flags =
-                (CGLO_GSTAT
-                 | (shouldCheckNumberOfBondedInteractions ? CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS : 0));
+        int cglo_flags = CGLO_GSTAT;
+        if (DOMAINDECOMP(cr) && shouldCheckNumberOfBondedInteractions(*cr->dd))
+        {
+            cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
+        }
         bool   bSumEkinhOld = false;
         t_vcm* vcm          = nullptr;
-        compute_globals(gstat, cr, ir, fr, ekind, makeConstArrayRef(state->x),
-                        makeConstArrayRef(state->v), state->box, mdatoms, nrnb, vcm, nullptr, enerd,
-                        force_vir, shake_vir, total_vir, pres, constr, &nullSignaller, state->box,
-                        &totalNumberOfBondedInteractions, &bSumEkinhOld, cglo_flags);
+        compute_globals(gstat,
+                        cr,
+                        ir,
+                        fr,
+                        ekind,
+                        makeConstArrayRef(state->x),
+                        makeConstArrayRef(state->v),
+                        state->box,
+                        mdatoms,
+                        nrnb,
+                        vcm,
+                        nullptr,
+                        enerd,
+                        force_vir,
+                        shake_vir,
+                        total_vir,
+                        pres,
+                        gmx::ArrayRef<real>{},
+                        &nullSignaller,
+                        state->box,
+                        &bSumEkinhOld,
+                        cglo_flags);
+        if (DOMAINDECOMP(cr))
+        {
+            checkNumberOfBondedInteractions(
+                    mdlog, cr, top_global, &top, makeConstArrayRef(state->x), state->box);
+        }
     }
-    checkNumberOfBondedInteractions(mdlog, cr, totalNumberOfBondedInteractions, top_global, &top,
-                                    makeConstArrayRef(state->x), state->box,
-                                    &shouldCheckNumberOfBondedInteractions);
 
     if (MASTER(cr))
     {
         fprintf(stderr,
                 "starting md rerun '%s', reading coordinates from"
                 " input trajectory '%s'\n\n",
-                *(top_global->name), opt2fn("-rerun", nfile, fnm));
+                *(top_global.name),
+                opt2fn("-rerun", nfile, fnm));
         if (mdrunOptions.verbose)
         {
             fprintf(stderr,
@@ -377,7 +439,7 @@ void gmx::LegacySimulator::do_rerun()
     }
 
     walltime_accounting_start_time(walltime_accounting);
-    wallcycle_start(wcycle, ewcRUN);
+    wallcycle_start(wcycle, WallCycleCounter::Run);
     print_start(fplog, cr, walltime_accounting, "mdrun");
 
     /***********************************************************
@@ -397,12 +459,13 @@ void gmx::LegacySimulator::do_rerun()
     if (MASTER(cr))
     {
         isLastStep = !read_first_frame(oenv, &status, opt2fn("-rerun", nfile, fnm), &rerun_fr, TRX_NEED_X);
-        if (rerun_fr.natoms != top_global->natoms)
+        if (rerun_fr.natoms != top_global.natoms)
         {
             gmx_fatal(FARGS,
                       "Number of atoms in trajectory (%d) does not match the "
                       "run input file (%d)\n",
-                      rerun_fr.natoms, top_global->natoms);
+                      rerun_fr.natoms,
+                      top_global.natoms);
         }
 
         if (ir->pbcType != PbcType::No)
@@ -413,7 +476,8 @@ void gmx::LegacySimulator::do_rerun()
                           "Rerun trajectory frame step %" PRId64
                           " time %f "
                           "does not contain a box, while pbc is used",
-                          rerun_fr.step, rerun_fr.time);
+                          rerun_fr.step,
+                          rerun_fr.time);
             }
             if (max_cutoff2(ir->pbcType, rerun_fr.box) < gmx::square(fr->rlist))
             {
@@ -421,7 +485,8 @@ void gmx::LegacySimulator::do_rerun()
                           "Rerun trajectory frame step %" PRId64
                           " time %f "
                           "has too small box dimensions",
-                          rerun_fr.step, rerun_fr.time);
+                          rerun_fr.step,
+                          rerun_fr.time);
             }
         }
     }
@@ -449,9 +514,18 @@ void gmx::LegacySimulator::do_rerun()
     step_rel = 0;
 
     auto stopHandler = stopHandlerBuilder->getStopHandlerMD(
-            compat::not_null<SimulationSignal*>(&signals[eglsSTOPCOND]), false, MASTER(cr),
-            ir->nstlist, mdrunOptions.reproducible, nstglobalcomm, mdrunOptions.maximumHoursToRun,
-            ir->nstlist == 0, fplog, step, bNS, walltime_accounting);
+            compat::not_null<SimulationSignal*>(&signals[eglsSTOPCOND]),
+            false,
+            MASTER(cr),
+            ir->nstlist,
+            mdrunOptions.reproducible,
+            nstglobalcomm,
+            mdrunOptions.maximumHoursToRun,
+            ir->nstlist == 0,
+            fplog,
+            step,
+            bNS,
+            walltime_accounting);
 
     // we don't do counter resetting in rerun - finish will always be valid
     walltime_accounting_set_valid_finish(walltime_accounting);
@@ -462,7 +536,7 @@ void gmx::LegacySimulator::do_rerun()
     isLastStep = (isLastStep || (ir->nsteps >= 0 && step_rel > ir->nsteps));
     while (!isLastStep)
     {
-        wallcycle_start(wcycle, ewcSTEP);
+        wallcycle_start(wcycle, WallCycleCounter::Step);
 
         if (rerun_fr.bStep)
         {
@@ -478,7 +552,7 @@ void gmx::LegacySimulator::do_rerun()
             t = step;
         }
 
-        if (ir->efep != efepNO && MASTER(cr))
+        if (ir->efep != FreeEnergyPerturbationType::No && MASTER(cr))
         {
             if (rerun_fr.bLambda)
             {
@@ -505,7 +579,7 @@ void gmx::LegacySimulator::do_rerun()
                           "decomposition, "
                           "use a single rank");
             }
-            prepareRerunState(rerun_fr, state_global, constructVsites, vsite, ir->delta_t);
+            prepareRerunState(rerun_fr, state_global, constructVsites, vsite);
         }
 
         isLastStep = isLastStep || stopHandler->stoppingAfterCurrentStep(bNS);
@@ -514,10 +588,27 @@ void gmx::LegacySimulator::do_rerun()
         {
             /* Repartition the domain decomposition */
             const bool bMasterState = true;
-            dd_partition_system(fplog, mdlog, step, cr, bMasterState, nstglobalcomm, state_global,
-                                *top_global, ir, imdSession, pull_work, state, &f, mdAtoms, &top,
-                                fr, vsite, constr, nrnb, wcycle, mdrunOptions.verbose);
-            shouldCheckNumberOfBondedInteractions = true;
+            dd_partition_system(fplog,
+                                mdlog,
+                                step,
+                                cr,
+                                bMasterState,
+                                nstglobalcomm,
+                                state_global,
+                                top_global,
+                                *ir,
+                                imdSession,
+                                pull_work,
+                                state,
+                                &f,
+                                mdAtoms,
+                                &top,
+                                fr,
+                                vsite,
+                                constr,
+                                nrnb,
+                                wcycle,
+                                mdrunOptions.verbose);
         }
 
         if (MASTER(cr))
@@ -525,9 +616,9 @@ void gmx::LegacySimulator::do_rerun()
             EnergyOutput::printHeader(fplog, step, t); /* can we improve the information printed here? */
         }
 
-        if (ir->efep != efepNO)
+        if (ir->efep != FreeEnergyPerturbationType::No)
         {
-            update_mdatoms(mdatoms, state->lambda[efptMASS]);
+            update_mdatoms(mdatoms, state->lambda[FreeEnergyPerturbationCouplingType::Mass]);
         }
 
         force_flags = (GMX_FORCE_STATECHANGED | GMX_FORCE_DYNAMICBOX | GMX_FORCE_ALLFORCES
@@ -537,12 +628,38 @@ void gmx::LegacySimulator::do_rerun()
         if (shellfc)
         {
             /* Now is the time to relax the shells */
-            relax_shell_flexcon(fplog, cr, ms, mdrunOptions.verbose, enforcedRotation, step, ir,
-                                imdSession, pull_work, bNS, force_flags, &top, constr, enerd,
-                                state->natoms, state->x.arrayRefWithPadding(),
-                                state->v.arrayRefWithPadding(), state->box, state->lambda,
-                                &state->hist, &f.view(), force_vir, mdatoms, nrnb, wcycle, shellfc,
-                                fr, runScheduleWork, t, mu_tot, vsite, ddBalanceRegionHandler);
+            relax_shell_flexcon(fplog,
+                                cr,
+                                ms,
+                                mdrunOptions.verbose,
+                                enforcedRotation,
+                                step,
+                                ir,
+                                imdSession,
+                                pull_work,
+                                bNS,
+                                force_flags,
+                                &top,
+                                constr,
+                                enerd,
+                                state->natoms,
+                                state->x.arrayRefWithPadding(),
+                                state->v.arrayRefWithPadding(),
+                                state->box,
+                                state->lambda,
+                                &state->hist,
+                                &f.view(),
+                                force_vir,
+                                *mdatoms,
+                                nrnb,
+                                wcycle,
+                                shellfc,
+                                fr,
+                                runScheduleWork,
+                                t,
+                                mu_tot,
+                                vsite,
+                                ddBalanceRegionHandler);
         }
         else
         {
@@ -553,10 +670,34 @@ void gmx::LegacySimulator::do_rerun()
              */
             Awh*       awh = nullptr;
             gmx_edsam* ed  = nullptr;
-            do_force(fplog, cr, ms, ir, awh, enforcedRotation, imdSession, pull_work, step, nrnb,
-                     wcycle, &top, state->box, state->x.arrayRefWithPadding(), &state->hist,
-                     &f.view(), force_vir, mdatoms, enerd, state->lambda, fr, runScheduleWork,
-                     vsite, mu_tot, t, ed, GMX_FORCE_NS | force_flags, ddBalanceRegionHandler);
+            do_force(fplog,
+                     cr,
+                     ms,
+                     *ir,
+                     awh,
+                     enforcedRotation,
+                     imdSession,
+                     pull_work,
+                     step,
+                     nrnb,
+                     wcycle,
+                     &top,
+                     state->box,
+                     state->x.arrayRefWithPadding(),
+                     &state->hist,
+                     &f.view(),
+                     force_vir,
+                     mdatoms,
+                     enerd,
+                     state->lambda,
+                     fr,
+                     runScheduleWork,
+                     vsite,
+                     mu_tot,
+                     t,
+                     ed,
+                     GMX_FORCE_NS | force_flags,
+                     ddBalanceRegionHandler);
         }
 
         /* Now we have the energies and forces corresponding to the
@@ -566,21 +707,32 @@ void gmx::LegacySimulator::do_rerun()
             const bool isCheckpointingStep = false;
             const bool doRerun             = true;
             const bool bSumEkinhOld        = false;
-            do_md_trajectory_writing(fplog, cr, nfile, fnm, step, step_rel, t, ir, state,
-                                     state_global, observablesHistory, top_global, fr, outf,
-                                     energyOutput, ekind, f.view().force(), isCheckpointingStep,
-                                     doRerun, isLastStep, mdrunOptions.writeConfout, bSumEkinhOld);
+            do_md_trajectory_writing(fplog,
+                                     cr,
+                                     nfile,
+                                     fnm,
+                                     step,
+                                     step_rel,
+                                     t,
+                                     ir,
+                                     state,
+                                     state_global,
+                                     observablesHistory,
+                                     top_global,
+                                     fr,
+                                     outf,
+                                     energyOutput,
+                                     ekind,
+                                     f.view().force(),
+                                     isCheckpointingStep,
+                                     doRerun,
+                                     isLastStep,
+                                     mdrunOptions.writeConfout,
+                                     bSumEkinhOld);
         }
 
         stopHandler->setSignal();
 
-        if (vsite != nullptr)
-        {
-            wallcycle_start(wcycle, ewcVSITECONSTR);
-            vsite->construct(state->x, ir->delta_t, state->v, state->box);
-            wallcycle_stop(wcycle, ewcVSITECONSTR);
-        }
-
         {
             const bool          doInterSimSignal = false;
             const bool          doIntraSimSignal = true;
@@ -588,16 +740,38 @@ void gmx::LegacySimulator::do_rerun()
             t_vcm*              vcm              = nullptr;
             SimulationSignaller signaller(&signals, cr, ms, doInterSimSignal, doIntraSimSignal);
 
-            compute_globals(gstat, cr, ir, fr, ekind, makeConstArrayRef(state->x),
-                            makeConstArrayRef(state->v), state->box, mdatoms, nrnb, vcm, wcycle,
-                            enerd, force_vir, shake_vir, total_vir, pres, constr, &signaller,
-                            state->box, &totalNumberOfBondedInteractions, &bSumEkinhOld,
-                            CGLO_GSTAT | CGLO_ENERGY
-                                    | (shouldCheckNumberOfBondedInteractions ? CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS
-                                                                             : 0));
-            checkNumberOfBondedInteractions(mdlog, cr, totalNumberOfBondedInteractions, top_global,
-                                            &top, makeConstArrayRef(state->x), state->box,
-                                            &shouldCheckNumberOfBondedInteractions);
+            int cglo_flags = CGLO_GSTAT | CGLO_ENERGY;
+            if (DOMAINDECOMP(cr) && shouldCheckNumberOfBondedInteractions(*cr->dd))
+            {
+                cglo_flags |= CGLO_CHECK_NUMBER_OF_BONDED_INTERACTIONS;
+            }
+            compute_globals(gstat,
+                            cr,
+                            ir,
+                            fr,
+                            ekind,
+                            makeConstArrayRef(state->x),
+                            makeConstArrayRef(state->v),
+                            state->box,
+                            mdatoms,
+                            nrnb,
+                            vcm,
+                            wcycle,
+                            enerd,
+                            force_vir,
+                            shake_vir,
+                            total_vir,
+                            pres,
+                            constr != nullptr ? constr->rmsdData() : gmx::ArrayRef<real>{},
+                            &signaller,
+                            state->box,
+                            &bSumEkinhOld,
+                            cglo_flags);
+            if (DOMAINDECOMP(cr))
+            {
+                checkNumberOfBondedInteractions(
+                        mdlog, cr, top_global, &top, makeConstArrayRef(state->x), state->box);
+            }
         }
 
         /* Note: this is OK, but there are some numerical precision issues with using the convergence of
@@ -609,12 +783,27 @@ void gmx::LegacySimulator::do_rerun()
         if (MASTER(cr))
         {
             const bool bCalcEnerStep = true;
-            energyOutput.addDataAtEnergyStep(
-                    doFreeEnergyPerturbation, bCalcEnerStep, t, mdatoms->tmass, enerd, ir->fepvals,
-                    ir->expandedvals, state->box,
-                    PTCouplingArrays({ state->boxv, state->nosehoover_xi, state->nosehoover_vxi,
-                                       state->nhpres_xi, state->nhpres_vxi }),
-                    state->fep_state, shake_vir, force_vir, total_vir, pres, ekind, mu_tot, constr);
+            energyOutput.addDataAtEnergyStep(doFreeEnergyPerturbation,
+                                             bCalcEnerStep,
+                                             t,
+                                             mdatoms->tmass,
+                                             enerd,
+                                             ir->fepvals.get(),
+                                             ir->expandedvals.get(),
+                                             state->box,
+                                             PTCouplingArrays({ state->boxv,
+                                                                state->nosehoover_xi,
+                                                                state->nosehoover_vxi,
+                                                                state->nhpres_xi,
+                                                                state->nhpres_vxi }),
+                                             state->fep_state,
+                                             shake_vir,
+                                             force_vir,
+                                             total_vir,
+                                             pres,
+                                             ekind,
+                                             mu_tot,
+                                             constr);
 
             const bool do_ene = true;
             const bool do_log = true;
@@ -623,8 +812,15 @@ void gmx::LegacySimulator::do_rerun()
             const bool do_or  = ir->nstorireout != 0;
 
             EnergyOutput::printAnnealingTemperatures(do_log ? fplog : nullptr, groups, &(ir->opts));
-            energyOutput.printStepToEnergyFile(mdoutf_get_fp_ene(outf), do_ene, do_dr, do_or,
-                                               do_log ? fplog : nullptr, step, t, fr->fcdata.get(), awh);
+            energyOutput.printStepToEnergyFile(mdoutf_get_fp_ene(outf),
+                                               do_ene,
+                                               do_dr,
+                                               do_or,
+                                               do_log ? fplog : nullptr,
+                                               step,
+                                               t,
+                                               fr->fcdata.get(),
+                                               awh);
 
             if (ir->bPull)
             {
@@ -653,11 +849,20 @@ void gmx::LegacySimulator::do_rerun()
         /* Ion/water position swapping.
          * Not done in last step since trajectory writing happens before this call
          * in the MD loop and exchanges would be lost anyway. */
-        if ((ir->eSwapCoords != eswapNO) && (step > 0) && !isLastStep && do_per_step(step, ir->swap->nstswap))
+        if ((ir->eSwapCoords != SwapType::No) && (step > 0) && !isLastStep
+            && do_per_step(step, ir->swap->nstswap))
         {
             const bool doRerun = true;
-            do_swapcoords(cr, step, t, ir, swap, wcycle, rerun_fr.x, rerun_fr.box,
-                          MASTER(cr) && mdrunOptions.verbose, doRerun);
+            do_swapcoords(cr,
+                          step,
+                          t,
+                          ir,
+                          swap,
+                          wcycle,
+                          rerun_fr.x,
+                          rerun_fr.box,
+                          MASTER(cr) && mdrunOptions.verbose,
+                          doRerun);
         }
 
         if (MASTER(cr))
@@ -671,7 +876,7 @@ void gmx::LegacySimulator::do_rerun()
             rerun_parallel_comm(cr, &rerun_fr, &isLastStep);
         }
 
-        cycles = wallcycle_stop(wcycle, ewcSTEP);
+        cycles = wallcycle_stop(wcycle, WallCycleCounter::Step);
         if (DOMAINDECOMP(cr) && wcycle)
         {
             dd_cycles_add(cr->dd, cycles, ddCyclStep);