Up to now, g_tune_pme would put the -nt switch on the mdrun command
line to ask for thread-MPI processes. Due to changes in mdrun, this
now leads to the fact that all runs with separate PME nodes exit
because the number of MPI threads must be explicitly specified with
the -ntmpi switch. Now the g_tune_pme -nt option is renamed to -ntmpi.
The remaining changes are only minor code cleanup / formatting things
plus getting rid of unused variables.
Change-Id: I7518799ef593d78d1cce1496852f0241e6d01755
if (diff < 0.0) diff = -diff;
if (diff < eps)
if (diff < 0.0) diff = -diff;
if (diff < eps)
iFound = eFoundNothing;
if (1 == nnodes)
iFound = eFoundNothing;
if (1 == nnodes)
iFound = eFoundDDStr; /* Skip some case statements */
iFound = eFoundDDStr; /* Skip some case statements */
while (fgets(line, STRLEN, fp) != NULL)
{
while (fgets(line, STRLEN, fp) != NULL)
{
sscanf(line, "Domain decomposition grid %d x %d x %d, separate PME nodes %d",
&(perfdata->nx), &(perfdata->ny), &(perfdata->nz), &npme);
if (perfdata->nPMEnodes == -1)
sscanf(line, "Domain decomposition grid %d x %d x %d, separate PME nodes %d",
&(perfdata->nx), &(perfdata->ny), &(perfdata->nz), &npme);
if (perfdata->nPMEnodes == -1)
perfdata->guessPME = npme;
perfdata->guessPME = npme;
else if (perfdata->nPMEnodes != npme)
else if (perfdata->nPMEnodes != npme)
gmx_fatal(FARGS, "PME nodes from command line and output file are not identical");
gmx_fatal(FARGS, "PME nodes from command line and output file are not identical");
iFound = eFoundDDStr;
}
/* Catch a few errors that might have occured: */
iFound = eFoundDDStr;
}
/* Catch a few errors that might have occured: */
case eFoundDDStr:
/* Look for PME mesh/force balance (not necessarily present, though) */
if (str_starts(line, matchstrbal))
case eFoundDDStr:
/* Look for PME mesh/force balance (not necessarily present, though) */
if (str_starts(line, matchstrbal))
sscanf(&line[strlen(matchstrbal)], "%f", &(perfdata->PME_f_load[test_nr]));
sscanf(&line[strlen(matchstrbal)], "%f", &(perfdata->PME_f_load[test_nr]));
/* Look for matchstring */
if (str_starts(line, matchstring))
/* Look for matchstring */
if (str_starts(line, matchstring))
iFound = eFoundAccountingStr;
iFound = eFoundAccountingStr;
break;
case eFoundAccountingStr:
/* Already found matchstring - look for cycle data */
break;
case eFoundAccountingStr:
/* Already found matchstring - look for cycle data */
perfdata->ns_per_day[test_nr] = (ndum==5)? dum3 : dum1;
fclose(fp);
if (bResetChecked || presteps == 0)
perfdata->ns_per_day[test_nr] = (ndum==5)? dum3 : dum1;
fclose(fp);
if (bResetChecked || presteps == 0)
return eParselogResetProblem;
return eParselogResetProblem;
if ( str_starts(line, "Fatal error:") )
{
if (fgets(line, STRLEN, fp) != NULL)
if ( str_starts(line, "Fatal error:") )
{
if (fgets(line, STRLEN, fp) != NULL)
fprintf(stderr, "\nWARNING: An error occured during this benchmark:\n"
"%s\n", line);
fprintf(stderr, "\nWARNING: An error occured during this benchmark:\n"
"%s\n", line);
fclose(fp);
cleandata(perfdata, test_nr);
return eParselogFatal;
fclose(fp);
cleandata(perfdata, test_nr);
return eParselogFatal;
fprintf(fp, "Summary of successful runs:\n");
fprintf(fp, "Line tpr PME nodes Gcycles Av. Std.dev. ns/day PME/f");
if (nnodes > 1)
fprintf(fp, "Summary of successful runs:\n");
fprintf(fp, "Line tpr PME nodes Gcycles Av. Std.dev. ns/day PME/f");
if (nnodes > 1)
pd->ns_per_day_Av = 0.0;
if (pd->nPMEnodes == -1)
pd->ns_per_day_Av = 0.0;
if (pd->nPMEnodes == -1)
sprintf(strbuf, "(%3d)", pd->guessPME);
sprintf(strbuf, "(%3d)", pd->guessPME);
/* Get the average run time of a setting */
for (j=0; j<nrepeats; j++)
/* Get the average run time of a setting */
for (j=0; j<nrepeats; j++)
for (j=0; j<nrepeats; j++)
{
if (pd->ns_per_day[j] > 0.0)
for (j=0; j<nrepeats; j++)
{
if (pd->ns_per_day[j] > 0.0)
pd->ns_per_day_Av += pd->ns_per_day[j];
pd->ns_per_day_Av += pd->ns_per_day[j];
else
{
/* Somehow the performance number was not aquired for this run,
else
{
/* Somehow the performance number was not aquired for this run,
/* Nicer output: */
if (pd->PME_f_load_Av > 0.0)
/* Nicer output: */
if (pd->PME_f_load_Av > 0.0)
sprintf(str_PME_f_load, "%12.3f", pd->PME_f_load_Av);
sprintf(str_PME_f_load, "%12.3f", pd->PME_f_load_Av);
sprintf(str_PME_f_load, "%s", " - ");
sprintf(str_PME_f_load, "%s", " - ");
/* We assume we had a successful run if both averages are positive */
/* We assume we had a successful run if both averages are positive */
/* Calculate the standard deviation */
s = 0.0;
for (j=0; j<nrepeats; j++)
/* Calculate the standard deviation */
s = 0.0;
for (j=0; j<nrepeats; j++)
s += pow( pd->Gcycles[j] - pd->Gcycles_Av, 2 );
s += pow( pd->Gcycles[j] - pd->Gcycles_Av, 2 );
s /= (nrepeats - 1);
s = sqrt(s);
s /= (nrepeats - 1);
s = sqrt(s);
line, k, pd->nPMEnodes, strbuf, pd->Gcycles_Av, s,
pd->ns_per_day_Av, str_PME_f_load);
if (nnodes > 1)
line, k, pd->nPMEnodes, strbuf, pd->Gcycles_Av, s,
pd->ns_per_day_Av, str_PME_f_load);
if (nnodes > 1)
fprintf(fp, " %3d %3d %3d", pd->nx, pd->ny, pd->nz);
fprintf(fp, " %3d %3d %3d", pd->nx, pd->ny, pd->nz);
fprintf(fp, "\n");
}
/* Store the index of the best run found so far in 'winner': */
fprintf(fp, "\n");
}
/* Store the index of the best run found so far in 'winner': */
gmx_fatal(FARGS, "None of the runs was successful! Check %s for problems.", fn);
gmx_fatal(FARGS, "None of the runs was successful! Check %s for problems.", fn);
{
/* We have optimized the number of PME-only nodes */
if (winPME == -1)
{
/* We have optimized the number of PME-only nodes */
if (winPME == -1)
sprintf(strbuf, "%s", "the automatic number of PME nodes");
sprintf(strbuf, "%s", "the automatic number of PME nodes");
sprintf(strbuf, "%d PME nodes", winPME);
sprintf(strbuf, "%d PME nodes", winPME);
}
fprintf(fp, "Best performance was achieved with %s", strbuf);
if ((nrepeats > 1) && (ntests > 1))
}
fprintf(fp, "Best performance was achieved with %s", strbuf);
if ((nrepeats > 1) && (ntests > 1))
fprintf(fp, " (see line %d)", line_win);
fprintf(fp, " (see line %d)", line_win);
fprintf(fp, "\n");
/* Only mention settings if they were modified: */
fprintf(fp, "\n");
/* Only mention settings if they were modified: */
fprintf(fp, " New Coulomb radius: %f nm (was %f nm)\n", info->rcoulomb[k_win], info->rcoulomb[0]);
fprintf(fp, " New Coulomb radius: %f nm (was %f nm)\n", info->rcoulomb[k_win], info->rcoulomb[0]);
fprintf(fp, " New Van der Waals radius: %f nm (was %f nm)\n", info->rvdw[k_win], info->rvdw[0]);
fprintf(fp, " New Van der Waals radius: %f nm (was %f nm)\n", info->rvdw[k_win], info->rvdw[0]);
fprintf(fp, " New Fourier grid xyz: %d %d %d (was %d %d %d)\n", info->nkx[k_win], info->nky[k_win], info->nkz[k_win],
info->nkx[0], info->nky[0], info->nkz[0]);
fprintf(fp, " New Fourier grid xyz: %d %d %d (was %d %d %d)\n", info->nkx[k_win], info->nky[k_win], info->nkz[k_win],
info->nkx[0], info->nky[0], info->nkz[0]);
if (bCanUseOrigTPR && ntprs > 1)
if (bCanUseOrigTPR && ntprs > 1)
fprintf(fp, "and original PME settings.\n");
fprintf(fp, "and original PME settings.\n");
if (!bThreads)
{
if ( (cp = getenv("MPIRUN")) != NULL)
if (!bThreads)
{
if ( (cp = getenv("MPIRUN")) != NULL)
*cmd_mpirun = strdup(cp);
*cmd_mpirun = strdup(cp);
*cmd_mpirun = strdup(def_mpirun);
*cmd_mpirun = strdup(def_mpirun);
}
if ( (cp = getenv("MDRUN" )) != NULL )
}
if ( (cp = getenv("MDRUN" )) != NULL )
*cmd_mdrun = strdup(def_mdrun);
*cmd_mdrun = strdup(def_mdrun);
/* If no simulations have to be performed, we are done here */
if (repeats <= 0)
/* If no simulations have to be performed, we are done here */
if (repeats <= 0)
/* Run a small test to see whether mpirun + mdrun work */
fprintf(stdout, "Making sure that mdrun can be executed. ");
/* Run a small test to see whether mpirun + mdrun work */
fprintf(stdout, "Making sure that mdrun can be executed. ");
if (cp2!=NULL)
{
if ( str_starts(line, match_mdrun) )
if (cp2!=NULL)
{
if ( str_starts(line, match_mdrun) )
if ( str_starts(line, match_mpi) )
if ( str_starts(line, match_mpi) )
FILE *fp, /* General log file */
gmx_bool bThreads, /* whether to use threads */
char *cmd_mpirun, /* Command for mpirun */
FILE *fp, /* General log file */
gmx_bool bThreads, /* whether to use threads */
char *cmd_mpirun, /* Command for mpirun */
- char *cmd_np, /* Switch for -np or -nt or empty */
+ char *cmd_np, /* Switch for -np or -ntmpi or empty */
char *cmd_mdrun, /* Command for mdrun */
char *args_for_mdrun, /* Arguments for mdrun */
const char *simulation_tpr, /* This tpr will be simulated */
char *cmd_mdrun, /* Command for mdrun */
char *args_for_mdrun, /* Arguments for mdrun */
const char *simulation_tpr, /* This tpr will be simulated */
- int nnodes, /* Number of nodes to run on */
int nPMEnodes) /* Number of PME nodes to use */
{
char *command;
int nPMEnodes) /* Number of PME nodes to use */
{
char *command;
gmx_large_int_t statesteps, /* Step counter in checkpoint file */
real rmin, /* Minimal Coulomb radius */
real rmax, /* Maximal Coulomb radius */
gmx_large_int_t statesteps, /* Step counter in checkpoint file */
real rmin, /* Minimal Coulomb radius */
real rmax, /* Maximal Coulomb radius */
- real bScaleRvdw, /* Scale rvdw along with rcoulomb */
+ real bScaleRvdw, /* Scale rvdw along with rcoulomb */
int *ntprs, /* No. of TPRs to write, each with a different
rcoulomb and fourierspacing */
t_inputinfo *info, /* Contains information about mdp file options */
int *ntprs, /* No. of TPRs to write, each with a different
rcoulomb and fourierspacing */
t_inputinfo *info, /* Contains information about mdp file options */
/* Check if some kind of PME was chosen */
if (EEL_PME(ir->coulombtype) == FALSE)
/* Check if some kind of PME was chosen */
if (EEL_PME(ir->coulombtype) == FALSE)
gmx_fatal(FARGS, "Can only do optimizations for simulations with %s electrostatics.",
EELTYPE(eelPME));
gmx_fatal(FARGS, "Can only do optimizations for simulations with %s electrostatics.",
EELTYPE(eelPME));
/* Check if rcoulomb == rlist, which is necessary for plain PME. */
if ( (ir->cutoff_scheme != ecutsVERLET) &&
/* Check if rcoulomb == rlist, which is necessary for plain PME. */
if ( (ir->cutoff_scheme != ecutsVERLET) &&
{
box_size[d] = 0;
for(i=0;i<DIM;i++)
{
box_size[d] = 0;
for(i=0;i<DIM;i++)
box_size[d] += state.box[d][i]*state.box[d][i];
box_size[d] += state.box[d][i]*state.box[d][i];
box_size[d] = sqrt(box_size[d]);
}
box_size[d] = sqrt(box_size[d]);
}
box_size[XX]/ir->nkx, box_size[YY]/ir->nky, box_size[ZZ]/ir->nkz);
fprintf(fp, " Van der Waals type : %s\n", EVDWTYPE(ir->vdwtype));
if (EVDW_SWITCHED(ir->vdwtype))
box_size[XX]/ir->nkx, box_size[YY]/ir->nky, box_size[ZZ]/ir->nkz);
fprintf(fp, " Van der Waals type : %s\n", EVDWTYPE(ir->vdwtype));
if (EVDW_SWITCHED(ir->vdwtype))
fprintf(fp, " rvdw_switch : %f nm\n", ir->rvdw_switch);
fprintf(fp, " rvdw_switch : %f nm\n", ir->rvdw_switch);
if (EPME_SWITCHED(ir->coulombtype))
if (EPME_SWITCHED(ir->coulombtype))
fprintf(fp, " rlist : %f nm\n", ir->rlist);
fprintf(fp, " rlist : %f nm\n", ir->rlist);
if (ir->rlistlong != max_cutoff(ir->rvdw,ir->rcoulomb))
if (ir->rlistlong != max_cutoff(ir->rvdw,ir->rcoulomb))
fprintf(fp, " rlistlong : %f nm\n", ir->rlistlong);
fprintf(fp, " rlistlong : %f nm\n", ir->rlistlong);
/* Print a descriptive line about the tpr settings tested */
fprintf(fp, "\nWill try these real/reciprocal workload settings:\n");
/* Print a descriptive line about the tpr settings tested */
fprintf(fp, "\nWill try these real/reciprocal workload settings:\n");
fprintf(fp, " nkx nky nkz");
fprintf(fp, " spacing");
if (evdwCUT == ir->vdwtype)
fprintf(fp, " nkx nky nkz");
fprintf(fp, " spacing");
if (evdwCUT == ir->vdwtype)
if (EPME_SWITCHED(ir->coulombtype))
if (EPME_SWITCHED(ir->coulombtype))
if ( ir->rlistlong != max_cutoff(ir->rlist,max_cutoff(ir->rvdw,ir->rcoulomb)) )
if ( ir->rlistlong != max_cutoff(ir->rlist,max_cutoff(ir->rvdw,ir->rcoulomb)) )
fprintf(fp, " rlistlong");
fprintf(fp, " rlistlong");
fprintf(fp, " tpr file\n");
/* Loop to create the requested number of tpr input files */
fprintf(fp, " tpr file\n");
/* Loop to create the requested number of tpr input files */
fprintf(stdout,"Writing benchmark tpr %s with nsteps=", fn_bench_tprs[j]);
fprintf(stdout, gmx_large_int_pfmt, ir->nsteps);
if (j > 0)
fprintf(stdout,"Writing benchmark tpr %s with nsteps=", fn_bench_tprs[j]);
fprintf(stdout, gmx_large_int_pfmt, ir->nsteps);
if (j > 0)
fprintf(stdout,", scaling factor %f\n", fac);
fprintf(stdout,", scaling factor %f\n", fac);
fprintf(stdout,", unmodified settings\n");
fprintf(stdout,", unmodified settings\n");
write_tpx_state(fn_bench_tprs[j],ir,&state,&mtop);
write_tpx_state(fn_bench_tprs[j],ir,&state,&mtop);
fprintf(fp, "%5d%5d%5d", ir->nkx, ir->nky, ir->nkz);
fprintf(fp, " %9f ", info->fsx[j]);
if (evdwCUT == ir->vdwtype)
fprintf(fp, "%5d%5d%5d", ir->nkx, ir->nky, ir->nkz);
fprintf(fp, " %9f ", info->fsx[j]);
if (evdwCUT == ir->vdwtype)
fprintf(fp, "%10f", ir->rvdw);
fprintf(fp, "%10f", ir->rvdw);
if (EPME_SWITCHED(ir->coulombtype))
if (EPME_SWITCHED(ir->coulombtype))
fprintf(fp, "%10f", ir->rlist);
fprintf(fp, "%10f", ir->rlist);
if ( info->rlistlong[0] != max_cutoff(info->rlist[0],max_cutoff(info->rvdw[0],info->rcoulomb[0])) )
if ( info->rlistlong[0] != max_cutoff(info->rlist[0],max_cutoff(info->rvdw[0],info->rcoulomb[0])) )
fprintf(fp, "%10f", ir->rlistlong);
fprintf(fp, "%10f", ir->rlistlong);
fprintf(fp, " %-14s\n",fn_bench_tprs[j]);
/* Make it clear to the user that some additional settings were modified */
fprintf(fp, " %-14s\n",fn_bench_tprs[j]);
/* Make it clear to the user that some additional settings were modified */
fprintf(fp, "\nNote that in addition to the Coulomb radius and the Fourier grid\n"
"other input settings were also changed (see table above).\n"
"Please check if the modified settings are appropriate.\n");
fprintf(fp, "\nNote that in addition to the Coulomb radius and the Fourier grid\n"
"other input settings were also changed (see table above).\n"
"Please check if the modified settings are appropriate.\n");
fflush(stdout);
fflush(fp);
sfree(ir);
fflush(stdout);
fflush(fp);
sfree(ir);
/* Give the log file a nice name so one can later see which parameters were used */
numstring[0] = '\0';
if (nr > 0)
/* Give the log file a nice name so one can later see which parameters were used */
numstring[0] = '\0';
if (nr > 0)
sprintf(numstring, "_%d", nr);
sprintf(numstring, "_%d", nr);
sprintf(newfilename, "%s_no%d_np%d_npme%d%s", opt2fn("-bg",nfile,fnm), k, nnodes, nPMEnodes, numstring);
if (gmx_fexist(opt2fn("-bg",nfile,fnm)))
{
sprintf(newfilename, "%s_no%d_np%d_npme%d%s", opt2fn("-bg",nfile,fnm), k, nnodes, nPMEnodes, numstring);
if (gmx_fexist(opt2fn("-bg",nfile,fnm)))
{
fn = opt2fn(opt, nfile, fnm);
numstring[0] = '\0';
if (nr > 0)
fn = opt2fn(opt, nfile, fnm);
numstring[0] = '\0';
if (nr > 0)
sprintf(numstring, "_%d", nr);
sprintf(numstring, "_%d", nr);
sprintf(newfilename, "%s_no%d_np%d_npme%d%s", fn, k, nnodes, nPMEnodes, numstring);
if (gmx_fexist(fn))
{
sprintf(newfilename, "%s_no%d_np%d_npme%d%s", fn, k, nnodes, nPMEnodes, numstring);
if (gmx_fexist(fn))
{
else /* "auto" or "range" */
{
if (nnodes <= 64)
else /* "auto" or "range" */
{
if (nnodes <= 64)
}
/* Calculate how many entries we could possibly have (in case of -npme all) */
}
/* Calculate how many entries we could possibly have (in case of -npme all) */
{
nlistmax = maxPMEnodes - minPMEnodes + 3;
if (0 == minPMEnodes)
{
nlistmax = maxPMEnodes - minPMEnodes + 3;
if (0 == minPMEnodes)
/* Now make the actual list which is at most of size nlist */
snew(*nPMEnodes, nlistmax);
/* Now make the actual list which is at most of size nlist */
snew(*nPMEnodes, nlistmax);
fprintf(stderr, "Will try the following %d different values for -npme:\n", *nentries);
for (i=0; i<*nentries-1; i++)
fprintf(stderr, "Will try the following %d different values for -npme:\n", *nentries);
for (i=0; i<*nentries-1; i++)
fprintf(stderr, "%d, ", (*nPMEnodes)[i]);
fprintf(stderr, "%d, ", (*nPMEnodes)[i]);
fprintf(stderr, "and %d (auto).\n", (*nPMEnodes)[*nentries-1]);
}
fprintf(stderr, "and %d (auto).\n", (*nPMEnodes)[*nentries-1]);
}
char *cmd_args_bench, /* arguments for mdrun in a string */
const t_filenm *fnm, /* List of filenames from command line */
int nfile, /* Number of files specified on the cmdl. */
char *cmd_args_bench, /* arguments for mdrun in a string */
const t_filenm *fnm, /* List of filenames from command line */
int nfile, /* Number of files specified on the cmdl. */
- int sim_part, /* For checkpointing */
int presteps, /* DLB equilibration steps, is checked */
gmx_large_int_t cpt_steps) /* Time step counter in the checkpoint */
{
int presteps, /* DLB equilibration steps, is checked */
gmx_large_int_t cpt_steps) /* Time step counter in the checkpoint */
{
/* To prevent that all benchmarks fail due to a show-stopper argument
* on the mdrun command line, we make a quick check with mdrun -h first */
if (bFirst)
/* To prevent that all benchmarks fail due to a show-stopper argument
* on the mdrun command line, we make a quick check with mdrun -h first */
if (bFirst)
make_sure_it_runs(pd->mdrun_cmd_line, cmdline_length, fp);
make_sure_it_runs(pd->mdrun_cmd_line, cmdline_length, fp);
bFirst = FALSE;
/* Do a benchmark simulation: */
if (repeats > 1)
bFirst = FALSE;
/* Do a benchmark simulation: */
if (repeats > 1)
sprintf(buf, ", pass %d/%d", nr+1, repeats);
sprintf(buf, ", pass %d/%d", nr+1, repeats);
fprintf(stdout, "\n=== Progress %2.0f%%, tpr %d/%d, run %d/%d%s:\n",
(100.0*count)/totaltests,
k+1, nr_tprs, i+1, *pmeentries, buf);
fprintf(stdout, "\n=== Progress %2.0f%%, tpr %d/%d, run %d/%d%s:\n",
(100.0*count)/totaltests,
k+1, nr_tprs, i+1, *pmeentries, buf);
ret = parse_logfile(opt2fn("-bg",nfile,fnm), opt2fn("-err",nfile,fnm),
pd, nr, presteps, cpt_steps, nnodes);
if ((presteps > 0) && (ret == eParselogResetProblem))
ret = parse_logfile(opt2fn("-bg",nfile,fnm), opt2fn("-err",nfile,fnm),
pd, nr, presteps, cpt_steps, nnodes);
if ((presteps > 0) && (ret == eParselogResetProblem))
sprintf(buf, "(%3d)", pd->guessPME);
sprintf(buf, "(%3d)", pd->guessPME);
/* Nicer output */
if (pd->PME_f_load[nr] > 0.0)
/* Nicer output */
if (pd->PME_f_load[nr] > 0.0)
sprintf(str_PME_f_load, "%12.3f", pd->PME_f_load[nr]);
sprintf(str_PME_f_load, "%12.3f", pd->PME_f_load[nr]);
sprintf(str_PME_f_load, "%s", " - ");
sprintf(str_PME_f_load, "%s", " - ");
/* Write the data we got to disk */
fprintf(fp, "%4d%s %12.3f %12.3f %s %s", pd->nPMEnodes,
buf, pd->Gcycles[nr], pd->ns_per_day[nr], str_PME_f_load, ParseLog[ret]);
if (! (ret==eParselogOK || ret==eParselogNoDDGrid || ret==eParselogNotFound) )
/* Write the data we got to disk */
fprintf(fp, "%4d%s %12.3f %12.3f %s %s", pd->nPMEnodes,
buf, pd->Gcycles[nr], pd->ns_per_day[nr], str_PME_f_load, ParseLog[ret]);
if (! (ret==eParselogOK || ret==eParselogNoDDGrid || ret==eParselogNotFound) )
fprintf(fp, " Check %s file for problems.", ret==eParselogFatal? "err":"log");
fprintf(fp, " Check %s file for problems.", ret==eParselogFatal? "err":"log");
fprintf(fp, "\n");
fflush(fp);
count++;
fprintf(fp, "\n");
fflush(fp);
count++;
/* Make sure the input file exists */
if (!gmx_fexist(opt2fn("-s",nfile,fnm)))
/* Make sure the input file exists */
if (!gmx_fexist(opt2fn("-s",nfile,fnm)))
gmx_fatal(FARGS, "File %s not found.", opt2fn("-s",nfile,fnm));
gmx_fatal(FARGS, "File %s not found.", opt2fn("-s",nfile,fnm));
/* Make sure that the checkpoint file is not overwritten during benchmarking */
if ( (0 == strcmp(opt2fn("-cpi",nfile,fnm), opt2fn("-bcpo",nfile,fnm)) ) && (sim_part > 1) )
/* Make sure that the checkpoint file is not overwritten during benchmarking */
if ( (0 == strcmp(opt2fn("-cpi",nfile,fnm), opt2fn("-bcpo",nfile,fnm)) ) && (sim_part > 1) )
gmx_fatal(FARGS, "Checkpoint input (-cpi) and benchmark checkpoint output (-bcpo) files must not be identical.\n"
"The checkpoint input file must not be overwritten during the benchmarks.\n");
gmx_fatal(FARGS, "Checkpoint input (-cpi) and benchmark checkpoint output (-bcpo) files must not be identical.\n"
"The checkpoint input file must not be overwritten during the benchmarks.\n");
/* Make sure that repeats is >= 0 (if == 0, only write tpr files) */
if (repeats < 0)
/* Make sure that repeats is >= 0 (if == 0, only write tpr files) */
if (repeats < 0)
gmx_fatal(FARGS, "Number of repeats < 0!");
gmx_fatal(FARGS, "Number of repeats < 0!");
/* Check number of nodes */
if (nnodes < 1)
/* Check number of nodes */
if (nnodes < 1)
gmx_fatal(FARGS, "Number of nodes/threads must be a positive integer.");
gmx_fatal(FARGS, "Number of nodes/threads must be a positive integer.");
/* Automatically choose -ntpr if not set */
if (*ntprs < 1)
{
if (nnodes < 16)
/* Automatically choose -ntpr if not set */
if (*ntprs < 1)
{
if (nnodes < 16)
fprintf(stderr, "Note: Choose ntpr>1 to shift PME load between real and reciprocal space.\n");
fprintf(stderr, "Note: Choose ntpr>1 to shift PME load between real and reciprocal space.\n");
}
/* Make shure that rmin <= rcoulomb <= rmax */
}
/* Make shure that rmin <= rcoulomb <= rmax */
- if (*rmin <= 0) *rmin = rcoulomb;
- if (*rmax <= 0) *rmax = rcoulomb;
+ if (*rmin <= 0)
+ {
+ *rmin = rcoulomb;
+ }
+ if (*rmax <= 0)
+ {
+ *rmax = rcoulomb;
+ }
if ( !(*rmin <= *rmax) )
{
gmx_fatal(FARGS, "Please choose the Coulomb radii such that rmin <= rmax.\n"
if ( !(*rmin <= *rmax) )
{
gmx_fatal(FARGS, "Please choose the Coulomb radii such that rmin <= rmax.\n"
old = *ntprs;
/* If one of rmin, rmax is set, we need 2 tpr files at minimum */
if ( !is_equal(*rmax, rcoulomb) || !is_equal(*rmin, rcoulomb) )
old = *ntprs;
/* If one of rmin, rmax is set, we need 2 tpr files at minimum */
if ( !is_equal(*rmax, rcoulomb) || !is_equal(*rmin, rcoulomb) )
/* If both rmin, rmax are set, we need 3 tpr files at minimum */
if ( !is_equal(*rmax, rcoulomb) && !is_equal(*rmin, rcoulomb) )
/* If both rmin, rmax are set, we need 3 tpr files at minimum */
if ( !is_equal(*rmax, rcoulomb) && !is_equal(*rmin, rcoulomb) )
/* Check whether max and min fraction are within required values */
if (maxPMEfraction > 0.5 || maxPMEfraction < 0)
/* Check whether max and min fraction are within required values */
if (maxPMEfraction > 0.5 || maxPMEfraction < 0)
gmx_fatal(FARGS, "-max must be between 0 and 0.5");
gmx_fatal(FARGS, "-max must be between 0 and 0.5");
if (minPMEfraction > 0.5 || minPMEfraction < 0)
if (minPMEfraction > 0.5 || minPMEfraction < 0)
gmx_fatal(FARGS, "-min must be between 0 and 0.5");
gmx_fatal(FARGS, "-min must be between 0 and 0.5");
if (maxPMEfraction < minPMEfraction)
if (maxPMEfraction < minPMEfraction)
gmx_fatal(FARGS, "-max must be larger or equal to -min");
gmx_fatal(FARGS, "-max must be larger or equal to -min");
/* Check whether the number of steps - if it was set - has a reasonable value */
if (bench_nsteps < 0)
/* Check whether the number of steps - if it was set - has a reasonable value */
if (bench_nsteps < 0)
gmx_fatal(FARGS, "Number of steps must be positive.");
gmx_fatal(FARGS, "Number of steps must be positive.");
if (bench_nsteps > 10000 || bench_nsteps < 100)
{
if (bench_nsteps > 10000 || bench_nsteps < 100)
{
if (*ntprs > 1)
{
if (*rmin/rcoulomb < 0.75 || *rmax/rcoulomb > 1.25)
if (*ntprs > 1)
{
if (*rmin/rcoulomb < 0.75 || *rmax/rcoulomb > 1.25)
fprintf(stderr, "WARNING: Applying extreme scaling factor. I hope you know what you are doing.\n");
fprintf(stderr, "WARNING: Applying extreme scaling factor. I hope you know what you are doing.\n");
}
/* If a fixed number of PME nodes is set we do rcoulomb and PME gird tuning
}
/* If a fixed number of PME nodes is set we do rcoulomb and PME gird tuning
{
/* Apart from the input .tpr, all files starting with "-b" are for
* _b_enchmark files exclusively */
{
/* Apart from the input .tpr, all files starting with "-b" are for
* _b_enchmark files exclusively */
- if (0 == strncmp(opt,"-s", 2)) return FALSE;
+ if (0 == strncmp(opt,"-s", 2))
+ {
+ return FALSE;
+ }
+
if (0 == strncmp(opt,"-b", 2) || 0 == strncmp(opt,"-s", 2))
{
if (!bOptional || bSet)
if (0 == strncmp(opt,"-b", 2) || 0 == strncmp(opt,"-s", 2))
{
if (!bOptional || bSet)
if (bSet) /* These are additional input files like -cpi -ei */
if (bSet) /* These are additional input files like -cpi -ei */
/* Create the command line for the benchmark as well as for the real run */
static void create_command_line_snippets(
/* Create the command line for the benchmark as well as for the real run */
static void create_command_line_snippets(
gmx_bool bAppendFiles,
gmx_bool bKeepAndNumCPT,
gmx_bool bResetHWay,
int presteps,
int nfile,
t_filenm fnm[],
gmx_bool bAppendFiles,
gmx_bool bKeepAndNumCPT,
gmx_bool bResetHWay,
int presteps,
int nfile,
t_filenm fnm[],
- int npargs,
- t_pargs *pa,
- const char *procstring, /* How to pass the number of processors to $MPIRUN */
- char *cmd_np[], /* Actual command line snippet, e.g. '-np <N>' */
char *cmd_args_bench[], /* command line arguments for benchmark runs */
char *cmd_args_launch[], /* command line arguments for simulation run */
char extra_args[]) /* Add this to the end of the command line */
char *cmd_args_bench[], /* command line arguments for benchmark runs */
char *cmd_args_launch[], /* command line arguments for simulation run */
char extra_args[]) /* Add this to the end of the command line */
/* All options starting with -b* need the 'b' removed,
* therefore overwrite strbuf */
if (0 == strncmp(opt, "-b", 2))
/* All options starting with -b* need the 'b' removed,
* therefore overwrite strbuf */
if (0 == strncmp(opt, "-b", 2))
sprintf(strbuf, "-%s %s ", &opt[2], name);
sprintf(strbuf, "-%s %s ", &opt[2], name);
add_to_string(cmd_args_bench, strbuf);
}
if ( is_launch_file(opt,opt2bSet(opt,nfile,fnm)) )
add_to_string(cmd_args_bench, strbuf);
}
if ( is_launch_file(opt,opt2bSet(opt,nfile,fnm)) )
add_to_string(cmd_args_launch, strbuf);
add_to_string(cmd_args_launch, strbuf);
}
add_to_string(cmd_args_bench , extra_args);
}
add_to_string(cmd_args_bench , extra_args);
/* Set option opt */
static void setopt(const char *opt,int nfile,t_filenm fnm[])
{
/* Set option opt */
static void setopt(const char *opt,int nfile,t_filenm fnm[])
{
- for(i=0; (i<nfile); i++)
- if (strcmp(opt,fnm[i].opt)==0)
- fnm[i].flag |= ffSET;
+ for(i=0; (i<nfile); i++)
+ {
+ if (strcmp(opt,fnm[i].opt)==0)
+ {
+ fnm[i].flag |= ffSET;
+ }
+ }
int gmx_tune_pme(int argc,char *argv[])
{
const char *desc[] = {
int gmx_tune_pme(int argc,char *argv[])
{
const char *desc[] = {
- "For a given number [TT]-np[tt] or [TT]-nt[tt] of processors/threads, this program systematically",
+ "For a given number [TT]-np[tt] or [TT]-ntmpi[tt] of processors/threads, this program systematically",
"times [TT]mdrun[tt] with various numbers of PME-only nodes and determines",
"which setting is fastest. It will also test whether performance can",
"be enhanced by shifting load from the reciprocal to the real space",
"times [TT]mdrun[tt] with various numbers of PME-only nodes and determines",
"which setting is fastest. It will also test whether performance can",
"be enhanced by shifting load from the reciprocal to the real space",
"[TT]export MPIRUN=\"/usr/local/mpirun -machinefile hosts\"[tt][PAR]",
"Please call [TT]g_tune_pme[tt] with the normal options you would pass to",
"[TT]mdrun[tt] and add [TT]-np[tt] for the number of processors to perform the",
"[TT]export MPIRUN=\"/usr/local/mpirun -machinefile hosts\"[tt][PAR]",
"Please call [TT]g_tune_pme[tt] with the normal options you would pass to",
"[TT]mdrun[tt] and add [TT]-np[tt] for the number of processors to perform the",
- "tests on, or [TT]-nt[tt] for the number of threads. You can also add [TT]-r[tt]",
+ "tests on, or [TT]-ntmpi[tt] for the number of threads. You can also add [TT]-r[tt]",
"to repeat each test several times to get better statistics. [PAR]",
"[TT]g_tune_pme[tt] can test various real space / reciprocal space workloads",
"for you. With [TT]-ntpr[tt] you control how many extra [TT].tpr[tt] files will be",
"to repeat each test several times to get better statistics. [PAR]",
"[TT]g_tune_pme[tt] can test various real space / reciprocal space workloads",
"for you. With [TT]-ntpr[tt] you control how many extra [TT].tpr[tt] files will be",
"Number of nodes to run the tests on (must be > 2 for separate PME nodes)" },
{ "-npstring", FALSE, etENUM, {procstring},
"Specify the number of processors to [TT]$MPIRUN[tt] using this string"},
"Number of nodes to run the tests on (must be > 2 for separate PME nodes)" },
{ "-npstring", FALSE, etENUM, {procstring},
"Specify the number of processors to [TT]$MPIRUN[tt] using this string"},
- { "-nt", FALSE, etINT, {&nthreads},
- "Number of threads to run the tests on (turns MPI & mpirun off)"},
+ { "-ntmpi", FALSE, etINT, {&nthreads},
+ "Number of MPI-threads to run the tests on (turns MPI & mpirun off)"},
{ "-r", FALSE, etINT, {&repeats},
"Repeat each test this often" },
{ "-max", FALSE, etREAL, {&maxPMEfraction},
{ "-r", FALSE, etINT, {&repeats},
"Repeat each test this often" },
{ "-max", FALSE, etREAL, {&maxPMEfraction},
add_to_string(&ExtraArgs, " ");
}
add_to_string(&ExtraArgs, " ");
}
- if (opt2parg_bSet("-nt",asize(pa),pa))
+ if (opt2parg_bSet("-ntmpi",asize(pa),pa))
{
bThreads=TRUE;
if (opt2parg_bSet("-npstring",asize(pa),pa))
{
bThreads=TRUE;
if (opt2parg_bSet("-npstring",asize(pa),pa))
fprintf(stderr, "WARNING: -npstring has no effect when using threads.\n");
fprintf(stderr, "WARNING: -npstring has no effect when using threads.\n");
gmx_fatal(FARGS, "Can't run multi-threaded MPI simulation yet!");
gmx_fatal(FARGS, "Can't run multi-threaded MPI simulation yet!");
/* and now we just set this; a bit of an ugly hack*/
nnodes=nthreads;
}
/* and now we just set this; a bit of an ugly hack*/
nnodes=nthreads;
}
/* Construct the command line arguments for benchmark runs
* as well as for the simulation run */
if (bThreads)
/* Construct the command line arguments for benchmark runs
* as well as for the simulation run */
if (bThreads)
- sprintf(bbuf," -nt %d ", nthreads);
+ {
+ sprintf(bbuf," -ntmpi %d ", nthreads);
+ }
sprintf(bbuf," -np %d ", nnodes);
sprintf(bbuf," -np %d ", nnodes);
- create_command_line_snippets(bThreads,bAppendFiles,bKeepAndNumCPT,bResetCountersHalfWay,presteps,
- NFILE,fnm,asize(pa),pa,procstring[0],
- &cmd_np, &cmd_args_bench, &cmd_args_launch,
- ExtraArgs);
+ create_command_line_snippets(bAppendFiles,bKeepAndNumCPT,bResetCountersHalfWay,presteps,
+ NFILE,fnm,&cmd_args_bench, &cmd_args_launch, ExtraArgs);
/* Read in checkpoint file if requested */
sim_part = 1;
/* Read in checkpoint file if requested */
sim_part = 1;
sim_part++;
/* sim_part will now be 1 if no checkpoint file was found */
if (sim_part<=1)
sim_part++;
/* sim_part will now be 1 if no checkpoint file was found */
if (sim_part<=1)
gmx_fatal(FARGS, "Checkpoint file %s not found!", opt2fn("-cpi",NFILE,fnm));
gmx_fatal(FARGS, "Checkpoint file %s not found!", opt2fn("-cpi",NFILE,fnm));
}
/* Open performance output file and write header info */
}
/* Open performance output file and write header info */
minPMEnodes = max(floor(minPMEfraction*nnodes), 0);
fprintf(stdout, "Will try runs with %d ", minPMEnodes);
if (maxPMEnodes != minPMEnodes)
minPMEnodes = max(floor(minPMEfraction*nnodes), 0);
fprintf(stdout, "Will try runs with %d ", minPMEnodes);
if (maxPMEnodes != minPMEnodes)
fprintf(stdout, "- %d ", maxPMEnodes);
fprintf(stdout, "- %d ", maxPMEnodes);
fprintf(stdout, "PME-only nodes.\n Note that the automatic number of PME-only nodes and no separate PME nodes are always tested.\n");
}
}
fprintf(stdout, "PME-only nodes.\n Note that the automatic number of PME-only nodes and no separate PME nodes are always tested.\n");
}
}
fprintf(fp, "Number of nodes : %d\n", nnodes);
fprintf(fp, "The mpirun command is : %s\n", cmd_mpirun);
if ( strcmp(procstring[0], "none") != 0)
fprintf(fp, "Number of nodes : %d\n", nnodes);
fprintf(fp, "The mpirun command is : %s\n", cmd_mpirun);
if ( strcmp(procstring[0], "none") != 0)
fprintf(fp, "Passing # of nodes via : %s\n", procstring[0]);
fprintf(fp, "Passing # of nodes via : %s\n", procstring[0]);
fprintf(fp, "Not setting number of nodes in system call\n");
fprintf(fp, "Not setting number of nodes in system call\n");
fprintf(fp, "Number of threads : %d\n", nnodes);
fprintf(fp, "Number of threads : %d\n", nnodes);
fprintf(fp, "The mdrun command is : %s\n", cmd_mdrun);
fprintf(fp, "mdrun args benchmarks : %s\n", cmd_args_bench);
fprintf(fp, "The mdrun command is : %s\n", cmd_mdrun);
fprintf(fp, "mdrun args benchmarks : %s\n", cmd_args_bench);
/* Make alternative tpr files to test: */
snew(tpr_names, ntprs);
for (i=0; i<ntprs; i++)
/* Make alternative tpr files to test: */
snew(tpr_names, ntprs);
for (i=0; i<ntprs; i++)
snew(tpr_names[i], STRLEN);
snew(tpr_names[i], STRLEN);
/* It can be that ntprs is reduced by make_benchmark_tprs if not enough
* different grids could be found. */
make_benchmark_tprs(opt2fn("-s",NFILE,fnm), tpr_names, bench_nsteps+presteps,
/* It can be that ntprs is reduced by make_benchmark_tprs if not enough
* different grids could be found. */
make_benchmark_tprs(opt2fn("-s",NFILE,fnm), tpr_names, bench_nsteps+presteps,
- cpt_steps, rmin, rmax, bScaleRvdw, &ntprs, info, fp);
+ cpt_steps, rmin, rmax, bScaleRvdw, &ntprs, info, fp);
/********************************************************************************/
/* Main loop over all scenarios we need to test: tpr files, PME nodes, repeats */
/********************************************************************************/
/* Main loop over all scenarios we need to test: tpr files, PME nodes, repeats */
{
do_the_tests(fp, tpr_names, maxPMEnodes, minPMEnodes, npme_fixed, npmevalues_opt[0], perfdata, &pmeentries,
repeats, nnodes, ntprs, bThreads, cmd_mpirun, cmd_np, cmd_mdrun,
{
do_the_tests(fp, tpr_names, maxPMEnodes, minPMEnodes, npme_fixed, npmevalues_opt[0], perfdata, &pmeentries,
repeats, nnodes, ntprs, bThreads, cmd_mpirun, cmd_np, cmd_mdrun,
- cmd_args_bench, fnm, NFILE, sim_part, presteps, cpt_steps);
+ cmd_args_bench, fnm, NFILE, presteps, cpt_steps);
fprintf(fp, "\nTuning took%8.1f minutes.\n", (gettime()-seconds)/60.0);
fprintf(fp, "\nTuning took%8.1f minutes.\n", (gettime()-seconds)/60.0);
/* Now start the real simulation if the user requested it ... */
launch_simulation(bLaunch, fp, bThreads, cmd_mpirun, cmd_np, cmd_mdrun,
/* Now start the real simulation if the user requested it ... */
launch_simulation(bLaunch, fp, bThreads, cmd_mpirun, cmd_np, cmd_mdrun,
- cmd_args_launch, simulation_tpr, nnodes, best_npme);
+ cmd_args_launch, simulation_tpr, best_npme);
}
ffclose(fp);
/* ... or simply print the performance results to screen: */
if (!bLaunch)
}
ffclose(fp);
/* ... or simply print the performance results to screen: */
if (!bLaunch)
finalize(opt2fn("-p", NFILE, fnm));
finalize(opt2fn("-p", NFILE, fnm));