2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team,
6 * check out http://www.gromacs.org for more information.
7 * Copyright (c) 2012, by the GROMACS development team, led by
8 * David van der Spoel, Berk Hess, Erik Lindahl, and including many
9 * others, as listed in the AUTHORS file in the top-level source
10 * directory and at http://www.gromacs.org.
12 * GROMACS is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public License
14 * as published by the Free Software Foundation; either version 2.1
15 * of the License, or (at your option) any later version.
17 * GROMACS is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Lesser General Public License for more details.
22 * You should have received a copy of the GNU Lesser General Public
23 * License along with GROMACS; if not, see
24 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
25 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
27 * If you want to redistribute modifications to GROMACS, please
28 * consider that scientific software is very special. Version
29 * control is crucial - bugs must be traceable. We will be happy to
30 * consider code for inclusion in the official distribution, but
31 * derived work must not be called official GROMACS. Details are found
32 * in the README & COPYING files - if they are missing, get the
33 * official version at http://www.gromacs.org.
35 * To help us fund GROMACS development, we humbly ask that you cite
36 * the research papers on the package. Check out http://www.gromacs.org.
41 #include "gmx_header_config.h"
49 #ifdef HAVE_SYS_TIME_H
54 /* windows-specific include for _chdir() */
60 #include "gmx_fatal.h"
71 #include "thread_mpi.h"
74 /* The source code in this file should be thread-safe.
75 Please keep it that way. */
82 #ifdef GMX_NATIVE_WINDOWS
87 /* Portable version of ctime_r implemented in src/gmxlib/string2.c, but we do not want it declared in public installed headers */
89 gmx_ctime_r(const time_t *clock,char *buf, int n);
95 static void par_fn(char *base,int ftp,const t_commrec *cr,
96 gmx_bool bAppendSimId,gmx_bool bAppendNodeId,
97 char buf[],int bufsize)
101 if((size_t)bufsize<(strlen(base)+10))
102 gmx_mem("Character buffer too small!");
104 /* Copy to buf, and strip extension */
106 buf[strlen(base) - strlen(ftp2ext(fn2ftp(base))) - 1] = '\0';
109 sprintf(buf+strlen(buf),"%d",cr->ms->sim);
113 sprintf(buf+strlen(buf),"%d",cr->nodeid);
117 /* Add extension again */
118 strcat(buf,(ftp == efTPX) ? "tpr" : (ftp == efEDR) ? "edr" : ftp2ext(ftp));
121 fprintf(debug, "node %d par_fn '%s'\n",cr->nodeid,buf);
122 if (fn2ftp(buf) == efLOG)
124 fprintf(debug,"log\n");
129 void check_multi_int(FILE *log,const gmx_multisim_t *ms,int val,
133 gmx_bool bCompatible;
136 fprintf(log,"Multi-checking %s ... ",name);
140 "check_multi_int called with a NULL communication pointer");
144 gmx_sumi_sim(ms->nsim,ibuf,ms);
147 for(p=1; p<ms->nsim; p++)
148 bCompatible = bCompatible && (ibuf[p-1] == ibuf[p]);
159 fprintf(log,"\n%s is not equal for all subsystems\n",name);
160 for(p=0; p<ms->nsim; p++)
161 fprintf(log," subsystem %d: %d\n",p,ibuf[p]);
163 gmx_fatal(FARGS,"The %d subsystems are not compatible\n",ms->nsim);
169 void check_multi_large_int(FILE *log,const gmx_multisim_t *ms,
170 gmx_large_int_t val, const char *name)
172 gmx_large_int_t *ibuf;
174 gmx_bool bCompatible;
177 fprintf(log,"Multi-checking %s ... ",name);
181 "check_multi_int called with a NULL communication pointer");
185 gmx_sumli_sim(ms->nsim,ibuf,ms);
188 for(p=1; p<ms->nsim; p++)
189 bCompatible = bCompatible && (ibuf[p-1] == ibuf[p]);
200 fprintf(log,"\n%s is not equal for all subsystems\n",name);
201 for(p=0; p<ms->nsim; p++)
204 /* first make the format string */
205 snprintf(strbuf, 255, " subsystem %%d: %s\n",
207 fprintf(log,strbuf,p,ibuf[p]);
210 gmx_fatal(FARGS,"The %d subsystems are not compatible\n",ms->nsim);
217 char *gmx_gethostname(char *name, size_t len)
221 gmx_incons("gmx_gethostname called with len<8");
224 if (gethostname(name, len-1) != 0)
226 strncpy(name, "unknown",8);
229 strncpy(name, "unknown",8);
236 void gmx_log_open(const char *lognm,const t_commrec *cr,gmx_bool bMasterOnly,
237 gmx_bool bAppendFiles, FILE** fplog)
240 char buf[256],host[256];
242 char timebuf[STRLEN];
248 /* Communicate the filename for logfile */
249 if (cr->nnodes > 1 && !bMasterOnly
250 #ifdef GMX_THREAD_MPI
251 /* With thread MPI the non-master log files are opened later
252 * when the files names are already known on all nodes.
260 len = strlen(lognm) + 1;
262 gmx_bcast(sizeof(len),&len,cr);
269 tmpnm=gmx_strdup(lognm);
271 gmx_bcast(len*sizeof(*tmpnm),tmpnm,cr);
275 tmpnm=gmx_strdup(lognm);
280 if (!bMasterOnly && !MASTER(cr))
282 /* Since log always ends with '.log' let's use this info */
283 par_fn(tmpnm,efLOG,cr,FALSE,!bMasterOnly,buf,255);
284 fp = gmx_fio_fopen(buf, bAppendFiles ? "a+" : "w+" );
286 else if (!bAppendFiles)
288 fp = gmx_fio_fopen(tmpnm, bAppendFiles ? "a+" : "w+" );
293 gmx_fatal_set_log_file(fp);
295 /* Get some machine parameters */
296 gmx_gethostname(host,256);
301 # ifdef GMX_NATIVE_WINDOWS
315 "-----------------------------------------------------------\n"
316 "Restarting from checkpoint, appending to previous log file.\n"
321 gmx_ctime_r(&t,timebuf,STRLEN);
324 "Log file opened on %s"
325 "Host: %s pid: %d nodeid: %d nnodes: %d\n",
326 timebuf,host,pid,cr->nodeid,cr->nnodes);
327 gmx_print_version_info(fp);
336 void gmx_log_close(FILE *fp)
339 gmx_fatal_set_log_file(NULL);
344 static void comm_args(const t_commrec *cr,int *argc,char ***argv)
349 gmx_bcast(sizeof(*argc),argc,cr);
353 fprintf(stderr,"NODEID=%d argc=%d\n",cr->nodeid,*argc);
354 for(i=0; (i<*argc); i++) {
356 len = strlen((*argv)[i])+1;
357 gmx_bcast(sizeof(len),&len,cr);
359 snew((*argv)[i],len);
360 /*gmx_bcast(len*sizeof((*argv)[i][0]),(*argv)[i],cr);*/
361 gmx_bcast(len*sizeof(char),(*argv)[i],cr);
366 void init_multisystem(t_commrec *cr,int nsim, char **multidirs,
367 int nfile, const t_filenm fnm[],gmx_bool bParFn)
370 int nnodes,nnodpersim,sim,i,ftp;
373 MPI_Group mpi_group_world;
380 gmx_fatal(FARGS,"This binary is compiled without MPI support, can not do multiple simulations.");
385 if (nnodes % nsim != 0)
387 gmx_fatal(FARGS,"The number of nodes (%d) is not a multiple of the number of simulations (%d)",nnodes,nsim);
390 nnodpersim = nnodes/nsim;
391 sim = cr->nodeid/nnodpersim;
395 fprintf(debug,"We have %d simulations, %d nodes per simulation, local simulation is %d\n",nsim,nnodpersim,sim);
403 /* Create a communicator for the master nodes */
405 for(i=0; i<ms->nsim; i++)
407 rank[i] = i*nnodpersim;
409 MPI_Comm_group(MPI_COMM_WORLD,&mpi_group_world);
410 MPI_Group_incl(mpi_group_world,nsim,rank,&ms->mpi_group_masters);
412 MPI_Comm_create(MPI_COMM_WORLD,ms->mpi_group_masters,
413 &ms->mpi_comm_masters);
415 #if !defined(GMX_THREAD_MPI) && !defined(MPI_IN_PLACE_EXISTS)
416 /* initialize the MPI_IN_PLACE replacement buffers */
422 ms->mpb->ibuf_alloc=0;
423 ms->mpb->libuf_alloc=0;
424 ms->mpb->fbuf_alloc=0;
425 ms->mpb->dbuf_alloc=0;
430 /* Reduce the intra-simulation communication */
431 cr->sim_nodeid = cr->nodeid % nnodpersim;
432 cr->nnodes = nnodpersim;
434 MPI_Comm_split(MPI_COMM_WORLD,sim,cr->sim_nodeid,&cr->mpi_comm_mysim);
435 cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
436 cr->nodeid = cr->sim_nodeid;
441 fprintf(debug,"This is simulation %d",cr->ms->sim);
444 fprintf(debug,", local number of nodes %d, local nodeid %d",
445 cr->nnodes,cr->sim_nodeid);
447 fprintf(debug,"\n\n");
455 fprintf(debug,"Changing to directory %s\n",multidirs[cr->ms->sim]);
457 if (chdir(multidirs[cr->ms->sim]) != 0)
459 gmx_fatal(FARGS, "Couldn't change directory to %s: %s",
460 multidirs[cr->ms->sim],
466 /* Patch output and tpx, cpt and rerun input file names */
467 for(i=0; (i<nfile); i++)
469 /* Because of possible multiple extensions per type we must look
470 * at the actual file name
472 if (is_output(&fnm[i]) ||
473 fnm[i].ftp == efTPX || fnm[i].ftp == efCPT ||
474 strcmp(fnm[i].opt,"-rerun") == 0)
476 ftp = fn2ftp(fnm[i].fns[0]);
477 par_fn(fnm[i].fns[0],ftp,cr,TRUE,FALSE,buf,255);
478 sfree(fnm[i].fns[0]);
479 fnm[i].fns[0] = gmx_strdup(buf);
485 t_commrec *init_par(int *argc,char ***argv_ptr)
494 argv = argv_ptr ? *argv_ptr : NULL;
496 #if defined GMX_MPI && !defined GMX_THREAD_MPI
497 cr->sim_nodeid = gmx_setup(argc,argv,&cr->nnodes);
499 if (!PAR(cr) && (cr->sim_nodeid != 0))
501 gmx_comm("(!PAR(cr) && (cr->sim_nodeid != 0))");
504 cr->mpi_comm_mysim = MPI_COMM_WORLD;
505 cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
507 /* These should never be accessed */
508 cr->mpi_comm_mysim = NULL;
509 cr->mpi_comm_mygroup = NULL;
514 cr->nodeid = cr->sim_nodeid;
516 cr->duty = (DUTY_PP | DUTY_PME);
518 /* Communicate arguments if parallel */
519 #ifndef GMX_THREAD_MPI
522 comm_args(cr,argc,argv_ptr);
524 #endif /* GMX_THREAD_MPI */
527 #if !defined(GMX_THREAD_MPI) && !defined(MPI_IN_PLACE_EXISTS)
528 /* initialize the MPI_IN_PLACE replacement buffers */
534 cr->mpb->ibuf_alloc=0;
535 cr->mpb->libuf_alloc=0;
536 cr->mpb->fbuf_alloc=0;
537 cr->mpb->dbuf_alloc=0;
544 t_commrec *init_par_threads(const t_commrec *cro)
546 #ifdef GMX_THREAD_MPI
550 /* make a thread-specific commrec */
552 /* now copy the whole thing, so settings like the number of PME nodes
556 /* and we start setting our own thread-specific values for things */
557 MPI_Initialized(&initialized);
560 gmx_comm("Initializing threads without comm");
562 /* once threads will be used together with MPI, we'll
563 fill the cr structure with distinct data here. This might even work: */
564 cr->sim_nodeid = gmx_setup(0,NULL, &cr->nnodes);
566 cr->mpi_comm_mysim = MPI_COMM_WORLD;
567 cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
568 cr->nodeid = cr->sim_nodeid;
569 cr->duty = (DUTY_PP | DUTY_PME);