1 /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
4 * This source code is part of
8 * GROningen MAchine for Chemical Simulations
11 * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
12 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
13 * Copyright (c) 2001-2004, The GROMACS development team,
14 * check out http://www.gromacs.org for more information.
16 * This program is free software; you can redistribute it and/or
17 * modify it under the terms of the GNU General Public License
18 * as published by the Free Software Foundation; either version 2
19 * of the License, or (at your option) any later version.
21 * If you want to redistribute modifications, please consider that
22 * scientific software is very special. Version control is crucial -
23 * bugs must be traceable. We will be happy to consider code for
24 * inclusion in the official distribution, but derived work must not
25 * be called official GROMACS. Details are found in the README & COPYING
26 * files - if they are missing, get the official version at www.gromacs.org.
28 * To help us fund GROMACS development, we humbly ask that you cite
29 * the papers on the package - you can find them in the top README file.
31 * For more info, check our website at http://www.gromacs.org
34 * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
42 #include <sys/syscall.h>
57 #include "pull_rotation.h"
71 #include "checkpoint.h"
72 #include "mtop_util.h"
73 #include "sighandler.h"
76 #include "pull_rotation.h"
92 #include "md_openmm.h"
101 gmx_integrator_t *func;
104 /* The array should match the eI array in include/types/enums.h */
105 #ifdef GMX_OPENMM /* FIXME do_md_openmm needs fixing */
106 const gmx_intp_t integrator[eiNR] = { {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm}, {do_md_openmm},{do_md_openmm}};
108 const gmx_intp_t integrator[eiNR] = { {do_md}, {do_steep}, {do_cg}, {do_md}, {do_md}, {do_nm}, {do_lbfgs}, {do_tpi}, {do_tpi}, {do_md}, {do_md},{do_md}};
111 gmx_large_int_t deform_init_init_step_tpx;
112 matrix deform_init_box_tpx;
113 #ifdef GMX_THREAD_MPI
114 tMPI_Thread_mutex_t deform_init_box_mutex=TMPI_THREAD_MUTEX_INITIALIZER;
118 #ifdef GMX_THREAD_MPI
119 struct mdrunner_arglist
133 const char *dddlb_opt;
146 const char *deviceOptions;
148 int ret; /* return value */
152 /* The function used for spawning threads. Extracts the mdrunner()
153 arguments from its one argument and calls mdrunner(), after making
155 static void mdrunner_start_fn(void *arg)
157 struct mdrunner_arglist *mda=(struct mdrunner_arglist*)arg;
158 struct mdrunner_arglist mc=*mda; /* copy the arg list to make sure
159 that it's thread-local. This doesn't
160 copy pointed-to items, of course,
161 but those are all const. */
162 t_commrec *cr; /* we need a local version of this */
166 fnm = dup_tfn(mc.nfile, mc.fnm);
168 cr = init_par_threads(mc.cr);
175 mda->ret=mdrunner(cr->nnodes, fplog, cr, mc.nfile, fnm, mc.oenv,
176 mc.bVerbose, mc.bCompact, mc.nstglobalcomm,
177 mc.ddxyz, mc.dd_node_order, mc.rdd,
178 mc.rconstr, mc.dddlb_opt, mc.dlb_scale,
179 mc.ddcsx, mc.ddcsy, mc.ddcsz, mc.nstepout, mc.resetstep,
180 mc.nmultisim, mc.repl_ex_nst, mc.repl_ex_seed, mc.pforce,
181 mc.cpt_period, mc.max_hours, mc.deviceOptions, mc.Flags);
184 /* called by mdrunner() to start a specific number of threads (including
185 the main thread) for thread-parallel runs. This in turn calls mdrunner()
187 All options besides nthreads are the same as for mdrunner(). */
188 static t_commrec *mdrunner_start_threads(int nthreads,
189 FILE *fplog,t_commrec *cr,int nfile,
190 const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
191 gmx_bool bCompact, int nstglobalcomm,
192 ivec ddxyz,int dd_node_order,real rdd,real rconstr,
193 const char *dddlb_opt,real dlb_scale,
194 const char *ddcsx,const char *ddcsy,const char *ddcsz,
195 int nstepout,int resetstep,int nmultisim,int repl_ex_nst,
196 int repl_ex_seed, real pforce,real cpt_period, real max_hours,
197 const char *deviceOptions, unsigned long Flags)
200 struct mdrunner_arglist *mda;
201 t_commrec *crn; /* the new commrec */
204 /* first check whether we even need to start tMPI */
208 /* a few small, one-time, almost unavoidable memory leaks: */
210 fnmn=dup_tfn(nfile, fnm);
212 /* fill the data structure to pass as void pointer to thread start fn */
218 mda->bVerbose=bVerbose;
219 mda->bCompact=bCompact;
220 mda->nstglobalcomm=nstglobalcomm;
221 mda->ddxyz[XX]=ddxyz[XX];
222 mda->ddxyz[YY]=ddxyz[YY];
223 mda->ddxyz[ZZ]=ddxyz[ZZ];
224 mda->dd_node_order=dd_node_order;
226 mda->rconstr=rconstr;
227 mda->dddlb_opt=dddlb_opt;
228 mda->dlb_scale=dlb_scale;
232 mda->nstepout=nstepout;
233 mda->resetstep=resetstep;
234 mda->nmultisim=nmultisim;
235 mda->repl_ex_nst=repl_ex_nst;
236 mda->repl_ex_seed=repl_ex_seed;
238 mda->cpt_period=cpt_period;
239 mda->max_hours=max_hours;
240 mda->deviceOptions=deviceOptions;
243 fprintf(stderr, "Starting %d threads\n",nthreads);
245 /* now spawn new threads that start mdrunner_start_fn(), while
246 the main thread returns */
247 ret=tMPI_Init_fn(TRUE, nthreads, mdrunner_start_fn, (void*)(mda) );
248 if (ret!=TMPI_SUCCESS)
251 /* make a new comm_rec to reflect the new situation */
252 crn=init_par_threads(cr);
257 /* Get the number of threads to use for thread-MPI based on how many
258 * were requested, which algorithms we're using,
259 * and how many particles there are.
261 static int get_nthreads_mpi(int nthreads_requested, t_inputrec *inputrec,
264 int nthreads,nthreads_new;
265 int min_atoms_per_thread;
268 nthreads = nthreads_requested;
270 /* determine # of hardware threads. */
271 if (nthreads_requested < 1)
273 if ((env = getenv("GMX_MAX_THREADS")) != NULL)
276 sscanf(env,"%d",&nthreads);
279 gmx_fatal(FARGS,"GMX_MAX_THREADS (%d) should be larger than 0",
285 nthreads = tMPI_Thread_get_hw_number();
289 if (inputrec->eI == eiNM || EI_TPI(inputrec->eI))
291 /* Steps are divided over the nodes iso splitting the atoms */
292 min_atoms_per_thread = 0;
296 min_atoms_per_thread = MIN_ATOMS_PER_THREAD;
299 /* Check if an algorithm does not support parallel simulation. */
301 ( inputrec->eI == eiLBFGS ||
302 inputrec->coulombtype == eelEWALD ) )
304 fprintf(stderr,"\nThe integration or electrostatics algorithm doesn't support parallel runs. Not starting any threads.\n");
307 else if (nthreads_requested < 1 &&
308 mtop->natoms/nthreads < min_atoms_per_thread)
310 /* the thread number was chosen automatically, but there are too many
311 threads (too few atoms per thread) */
312 nthreads_new = max(1,mtop->natoms/min_atoms_per_thread);
314 if (nthreads_new > 8 || (nthreads == 8 && nthreads_new > 4))
316 /* Use only multiples of 4 above 8 threads
317 * or with an 8-core processor
318 * (to avoid 6 threads on 8 core processors with 4 real cores).
320 nthreads_new = (nthreads_new/4)*4;
322 else if (nthreads_new > 4)
324 /* Avoid 5 or 7 threads */
325 nthreads_new = (nthreads_new/2)*2;
328 nthreads = nthreads_new;
330 fprintf(stderr,"\n");
331 fprintf(stderr,"NOTE: Parallelization is limited by the small number of atoms,\n");
332 fprintf(stderr," only starting %d threads.\n",nthreads);
333 fprintf(stderr," You can use the -nt option to optimize the number of threads.\n\n");
340 int mdrunner(int nthreads_requested, FILE *fplog,t_commrec *cr,int nfile,
341 const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
342 gmx_bool bCompact, int nstglobalcomm,
343 ivec ddxyz,int dd_node_order,real rdd,real rconstr,
344 const char *dddlb_opt,real dlb_scale,
345 const char *ddcsx,const char *ddcsy,const char *ddcsz,
346 int nstepout,int resetstep,int nmultisim,int repl_ex_nst,
347 int repl_ex_seed, real pforce,real cpt_period,real max_hours,
348 const char *deviceOptions, unsigned long Flags)
350 double nodetime=0,realtime;
351 t_inputrec *inputrec;
354 gmx_ddbox_t ddbox={0};
355 int npme_major,npme_minor;
358 gmx_mtop_t *mtop=NULL;
359 t_mdatoms *mdatoms=NULL;
363 gmx_pme_t *pmedata=NULL;
364 gmx_vsite_t *vsite=NULL;
366 int i,m,nChargePerturbed=-1,status,nalloc;
368 gmx_wallcycle_t wcycle;
369 gmx_bool bReadRNG,bReadEkin;
371 gmx_runtime_t runtime;
373 gmx_large_int_t reset_counters;
375 t_commrec *cr_old=cr;
378 gmx_membed_t *membed=NULL;
380 /* CAUTION: threads may be started later on in this function, so
381 cr doesn't reflect the final parallel state right now */
385 if (bVerbose && SIMMASTER(cr))
387 fprintf(stderr,"Getting Loaded...\n");
390 if (Flags & MD_APPENDFILES)
398 /* Read (nearly) all data required for the simulation */
399 read_tpx_state(ftp2fn(efTPX,nfile,fnm),inputrec,state,NULL,mtop);
401 /* NOW the threads will be started: */
402 #ifdef GMX_THREAD_MPI
403 nthreads_mpi = get_nthreads_mpi(nthreads_requested, inputrec, mtop);
405 if (nthreads_mpi > 1)
407 /* now start the threads. */
408 cr=mdrunner_start_threads(nthreads_mpi, fplog, cr_old, nfile, fnm,
409 oenv, bVerbose, bCompact, nstglobalcomm,
410 ddxyz, dd_node_order, rdd, rconstr,
411 dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
412 nstepout, resetstep, nmultisim,
413 repl_ex_nst, repl_ex_seed, pforce,
414 cpt_period, max_hours, deviceOptions,
416 /* the main thread continues here with a new cr. We don't deallocate
417 the old cr because other threads may still be reading it. */
420 gmx_comm("Failed to spawn threads");
425 /* END OF CAUTION: cr is now reliable */
427 /* g_membed initialisation *
428 * Because we change the mtop, init_membed is called before the init_parallel *
429 * (in case we ever want to make it run in parallel) */
430 if (opt2bSet("-membed",nfile,fnm))
432 fprintf(stderr,"Initializing membed");
434 init_membed(fplog,membed,nfile,fnm,mtop,inputrec,state,cr,&cpt_period);
439 /* now broadcast everything to the non-master nodes/threads: */
440 init_parallel(fplog, cr, inputrec, mtop);
444 pr_inputrec(fplog,0,"Input Parameters",inputrec,FALSE);
447 /* now make sure the state is initialized and propagated */
448 set_state_entries(state,inputrec,cr->nnodes);
450 /* A parallel command line option consistency check that we can
451 only do after any threads have started. */
453 (ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
456 "The -dd or -npme option request a parallel simulation, "
458 "but mdrun was compiled without threads or MPI enabled"
460 #ifdef GMX_THREAD_MPI
461 "but the number of threads (option -nt) is 1"
463 "but mdrun was not started through mpirun/mpiexec or only one process was requested through mpirun/mpiexec"
469 if ((Flags & MD_RERUN) &&
470 (EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
472 gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
475 if (can_use_allvsall(inputrec,mtop,TRUE,cr,fplog))
477 /* All-vs-all loops do not work with domain decomposition */
481 if (!EEL_PME(inputrec->coulombtype) || (Flags & MD_PARTDEC))
483 if (cr->npmenodes > 0)
485 if (!EEL_PME(inputrec->coulombtype))
487 gmx_fatal_collective(FARGS,cr,NULL,
488 "PME nodes are requested, but the system does not use PME electrostatics");
490 if (Flags & MD_PARTDEC)
492 gmx_fatal_collective(FARGS,cr,NULL,
493 "PME nodes are requested, but particle decomposition does not support separate PME nodes");
501 fcRegisterSteps(inputrec->nsteps,inputrec->init_step);
504 /* NMR restraints must be initialized before load_checkpoint,
505 * since with time averaging the history is added to t_state.
506 * For proper consistency check we therefore need to extend
508 * So the PME-only nodes (if present) will also initialize
509 * the distance restraints.
513 /* This needs to be called before read_checkpoint to extend the state */
514 init_disres(fplog,mtop,inputrec,cr,Flags & MD_PARTDEC,fcd,state);
516 if (gmx_mtop_ftype_count(mtop,F_ORIRES) > 0)
518 if (PAR(cr) && !(Flags & MD_PARTDEC))
520 gmx_fatal(FARGS,"Orientation restraints do not work (yet) with domain decomposition, use particle decomposition (mdrun option -pd)");
522 /* Orientation restraints */
525 init_orires(fplog,mtop,state->x,inputrec,cr->ms,&(fcd->orires),
530 if (DEFORM(*inputrec))
532 /* Store the deform reference box before reading the checkpoint */
535 copy_mat(state->box,box);
539 gmx_bcast(sizeof(box),box,cr);
541 /* Because we do not have the update struct available yet
542 * in which the reference values should be stored,
543 * we store them temporarily in static variables.
544 * This should be thread safe, since they are only written once
545 * and with identical values.
547 #ifdef GMX_THREAD_MPI
548 tMPI_Thread_mutex_lock(&deform_init_box_mutex);
550 deform_init_init_step_tpx = inputrec->init_step;
551 copy_mat(box,deform_init_box_tpx);
552 #ifdef GMX_THREAD_MPI
553 tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
557 if (opt2bSet("-cpi",nfile,fnm))
559 /* Check if checkpoint file exists before doing continuation.
560 * This way we can use identical input options for the first and subsequent runs...
562 if( gmx_fexist_master(opt2fn_master("-cpi",nfile,fnm,cr),cr) )
564 load_checkpoint(opt2fn_master("-cpi",nfile,fnm,cr),&fplog,
565 cr,Flags & MD_PARTDEC,ddxyz,
566 inputrec,state,&bReadRNG,&bReadEkin,
567 (Flags & MD_APPENDFILES),
568 (Flags & MD_APPENDFILESSET));
572 Flags |= MD_READ_RNG;
576 Flags |= MD_READ_EKIN;
581 if (((MASTER(cr) || (Flags & MD_SEPPOT)) && (Flags & MD_APPENDFILES))
582 #ifdef GMX_THREAD_MPI
583 /* With thread MPI only the master node/thread exists in mdrun.c,
584 * therefore non-master nodes need to open the "seppot" log file here.
586 || (!MASTER(cr) && (Flags & MD_SEPPOT))
590 gmx_log_open(ftp2fn(efLOG,nfile,fnm),cr,!(Flags & MD_SEPPOT),
596 copy_mat(state->box,box);
601 gmx_bcast(sizeof(box),box,cr);
604 /* Essential dynamics */
605 if (opt2bSet("-ei",nfile,fnm))
607 /* Open input and output files, allocate space for ED data structure */
608 ed = ed_open(nfile,fnm,Flags,cr);
611 if (bVerbose && SIMMASTER(cr))
613 fprintf(stderr,"Loaded with Money\n\n");
616 if (PAR(cr) && !((Flags & MD_PARTDEC) ||
617 EI_TPI(inputrec->eI) ||
618 inputrec->eI == eiNM))
620 cr->dd = init_domain_decomposition(fplog,cr,Flags,ddxyz,rdd,rconstr,
625 &ddbox,&npme_major,&npme_minor);
627 make_dd_communicators(fplog,cr,dd_node_order);
629 /* Set overallocation to avoid frequent reallocation of arrays */
630 set_over_alloc_dd(TRUE);
634 /* PME, if used, is done on all nodes with 1D decomposition */
636 cr->duty = (DUTY_PP | DUTY_PME);
639 if (!EI_TPI(inputrec->eI))
641 npme_major = cr->nnodes;
644 if (inputrec->ePBC == epbcSCREW)
647 "pbc=%s is only implemented with domain decomposition",
648 epbc_names[inputrec->ePBC]);
654 /* After possible communicator splitting in make_dd_communicators.
655 * we can set up the intra/inter node communication.
657 gmx_setup_nodecomm(fplog,cr);
660 /* get number of OpenMP/PME threads
661 * env variable should be read only on one node to make sure it is identical everywhere */
663 if (EEL_PME(inputrec->coulombtype))
668 if ((ptr=getenv("GMX_PME_NTHREADS")) != NULL)
670 sscanf(ptr,"%d",&nthreads_pme);
672 if (fplog != NULL && nthreads_pme > 1)
674 fprintf(fplog,"Using %d threads for PME\n",nthreads_pme);
679 gmx_bcast_sim(sizeof(nthreads_pme),&nthreads_pme,cr);
684 wcycle = wallcycle_init(fplog,resetstep,cr,nthreads_pme);
687 /* Master synchronizes its value of reset_counters with all nodes
688 * including PME only nodes */
689 reset_counters = wcycle_get_reset_counters(wcycle);
690 gmx_bcast_sim(sizeof(reset_counters),&reset_counters,cr);
691 wcycle_set_reset_counters(wcycle, reset_counters);
696 if (cr->duty & DUTY_PP)
698 /* For domain decomposition we allocate dynamically
699 * in dd_partition_system.
701 if (DOMAINDECOMP(cr))
703 bcast_state_setup(cr,state);
709 bcast_state(cr,state,TRUE);
713 /* Dihedral Restraints */
714 if (gmx_mtop_ftype_count(mtop,F_DIHRES) > 0)
716 init_dihres(fplog,mtop,inputrec,fcd);
719 /* Initiate forcerecord */
721 init_forcerec(fplog,oenv,fr,fcd,inputrec,mtop,cr,box,FALSE,
722 opt2fn("-table",nfile,fnm),
723 opt2fn("-tabletf",nfile,fnm),
724 opt2fn("-tablep",nfile,fnm),
725 opt2fn("-tableb",nfile,fnm),FALSE,pforce);
727 /* version for PCA_NOT_READ_NODE (see md.c) */
728 /*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
729 "nofile","nofile","nofile","nofile",FALSE,pforce);
731 fr->bSepDVDL = ((Flags & MD_SEPPOT) == MD_SEPPOT);
733 /* Initialize QM-MM */
736 init_QMMMrec(cr,box,mtop,inputrec,fr);
739 /* Initialize the mdatoms structure.
740 * mdatoms is not filled with atom data,
741 * as this can not be done now with domain decomposition.
743 mdatoms = init_mdatoms(fplog,mtop,inputrec->efep!=efepNO);
745 /* Initialize the virtual site communication */
746 vsite = init_vsite(mtop,cr);
748 calc_shifts(box,fr->shift_vec);
750 /* With periodic molecules the charge groups should be whole at start up
751 * and the virtual sites should not be far from their proper positions.
753 if (!inputrec->bContinuation && MASTER(cr) &&
754 !(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
756 /* Make molecules whole at start of run */
757 if (fr->ePBC != epbcNONE)
759 do_pbc_first_mtop(fplog,inputrec->ePBC,box,mtop,state->x);
763 /* Correct initial vsite positions are required
764 * for the initial distribution in the domain decomposition
765 * and for the initial shell prediction.
767 construct_vsites_mtop(fplog,vsite,mtop,state->x);
771 if (EEL_PME(fr->eeltype))
773 ewaldcoeff = fr->ewaldcoeff;
774 pmedata = &fr->pmedata;
783 /* This is a PME only node */
785 /* We don't need the state */
788 ewaldcoeff = calc_ewaldcoeff(inputrec->rcoulomb, inputrec->ewald_rtol);
792 /* Initiate PME if necessary,
793 * either on all nodes or on dedicated PME nodes only. */
794 if (EEL_PME(inputrec->coulombtype))
798 nChargePerturbed = mdatoms->nChargePerturbed;
800 if (cr->npmenodes > 0)
802 /* The PME only nodes need to know nChargePerturbed */
803 gmx_bcast_sim(sizeof(nChargePerturbed),&nChargePerturbed,cr);
807 /* Set CPU affinity. Can be important for performance.
808 On some systems (e.g. Cray) CPU Affinity is set by default.
809 But default assigning doesn't work (well) with only some ranks
810 having threads. This causes very low performance.
811 External tools have cumbersome syntax for setting affinity
812 in the case that only some ranks have threads.
813 Thus it is important that GROMACS sets the affinity internally at
814 if only PME is using threads.
822 MPI_Comm comm_intra; /* intra communicator (but different to nc.comm_intra includes PME nodes) */
823 MPI_Comm_split(MPI_COMM_WORLD,gmx_hostname_num(),gmx_node_rank(),&comm_intra);
824 int local_omp_nthreads = (cr->duty & DUTY_PME) ? nthreads_pme : 1; /* threads on this node */
825 MPI_Scan(&local_omp_nthreads,&core, 1, MPI_INT, MPI_SUM, comm_intra);
826 core-=local_omp_nthreads; /* make exclusive scan */
827 #pragma omp parallel firstprivate(core) num_threads(local_omp_nthreads)
831 core+=omp_get_thread_num();
833 sched_setaffinity((pid_t) syscall (SYS_gettid),sizeof(cpu_set_t),&mask);
838 #endif /*GMX_OPENMP*/
840 if (cr->duty & DUTY_PME)
842 status = gmx_pme_init(pmedata,cr,npme_major,npme_minor,inputrec,
843 mtop ? mtop->natoms : 0,nChargePerturbed,
844 (Flags & MD_REPRODUCIBLE),nthreads_pme);
847 gmx_fatal(FARGS,"Error %d initializing PME",status);
853 if (integrator[inputrec->eI].func == do_md
856 integrator[inputrec->eI].func == do_md_openmm
860 /* Turn on signal handling on all nodes */
862 * (A user signal from the PME nodes (if any)
863 * is communicated to the PP nodes.
865 signal_handler_install();
868 if (cr->duty & DUTY_PP)
870 if (inputrec->ePull != epullNO)
872 /* Initialize pull code */
873 init_pull(fplog,inputrec,nfile,fnm,mtop,cr,oenv,
874 EI_DYNAMICS(inputrec->eI) && MASTER(cr),Flags);
879 /* Initialize enforced rotation code */
880 init_rot(fplog,inputrec,nfile,fnm,cr,state->x,state->box,mtop,oenv,
884 constr = init_constraints(fplog,mtop,inputrec,ed,state,cr);
886 if (DOMAINDECOMP(cr))
888 dd_init_bondeds(fplog,cr->dd,mtop,vsite,constr,inputrec,
889 Flags & MD_DDBONDCHECK,fr->cginfo_mb);
891 set_dd_parameters(fplog,cr->dd,dlb_scale,inputrec,fr,&ddbox);
893 setup_dd_grid(fplog,cr->dd);
896 /* Now do whatever the user wants us to do (how flexible...) */
897 integrator[inputrec->eI].func(fplog,cr,nfile,fnm,
898 oenv,bVerbose,bCompact,
901 nstepout,inputrec,mtop,
903 mdatoms,nrnb,wcycle,ed,fr,
904 repl_ex_nst,repl_ex_seed,
906 cpt_period,max_hours,
911 if (inputrec->ePull != epullNO)
913 finish_pull(fplog,inputrec->pull);
918 finish_rot(fplog,inputrec->rot);
925 gmx_pmeonly(*pmedata,cr,nrnb,wcycle,ewaldcoeff,FALSE,inputrec);
928 if (EI_DYNAMICS(inputrec->eI) || EI_TPI(inputrec->eI))
930 /* Some timing stats */
933 if (runtime.proc == 0)
935 runtime.proc = runtime.real;
944 wallcycle_stop(wcycle,ewcRUN);
946 /* Finish up, write some stuff
947 * if rerunMD, don't write last frame again
949 finish_run(fplog,cr,ftp2fn(efSTO,nfile,fnm),
950 inputrec,nrnb,wcycle,&runtime,
951 EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
953 if (opt2bSet("-membed",nfile,fnm))
958 /* Does what it says */
959 print_date_and_time(fplog,cr->nodeid,"Finished mdrun",&runtime);
961 /* Close logfile already here if we were appending to it */
962 if (MASTER(cr) && (Flags & MD_APPENDFILES))
964 gmx_log_close(fplog);
967 rc=(int)gmx_get_stop_condition();
969 #ifdef GMX_THREAD_MPI
970 /* we need to join all threads. The sub-threads join when they
971 exit this function, but the master thread needs to be told to
973 if (PAR(cr) && MASTER(cr))