{
gmx_domdec_master_t *ma = NULL;
int buf2[2], *ibuf, i, ncg_home = 0, *cg = NULL, nat_home = 0;
- t_block *cgs_gl;
if (state_local->ddp_count == dd->comm->master_cg_ddp_count)
{
if (state_local->ddp_count == dd->ddp_count)
{
+ /* The local state and DD are in sync, use the DD indices */
ncg_home = dd->ncg_home;
cg = dd->index_gl;
nat_home = dd->nat_home;
}
else if (state_local->ddp_count_cg_gl == state_local->ddp_count)
{
+ /* The DD is out of sync with the local state, but we have stored
+ * the cg indices with the local state, so we can use those.
+ */
+ t_block *cgs_gl;
+
cgs_gl = &dd->comm->cgs_gl;
ncg_home = state_local->ncg_gl;
gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
}
- buf2[0] = dd->ncg_home;
- buf2[1] = dd->nat_home;
+ buf2[0] = ncg_home;
+ buf2[1] = nat_home;
if (DDMASTER(dd))
{
ma = dd->ma;
/* Collect the charge group indices on the master */
dd_gatherv(dd,
- dd->ncg_home*sizeof(int), dd->index_gl,
+ ncg_home*sizeof(int), cg,
DDMASTER(dd) ? ma->ibuf : NULL,
DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
DDMASTER(dd) ? ma->cg : NULL);
if (debug)
{
- fprintf(debug, "Receive coordinates from PP nodes:");
+ fprintf(debug, "Receive coordinates from PP ranks:");
for (x = 0; x < *nmy_ddnodes; x++)
{
fprintf(debug, " %d", (*my_ddnodes)[x]);
if (!bLocalCG[dd->index_gl[i]])
{
fprintf(stderr,
- "DD node %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd->rank, where, i+1, dd->index_gl[i]+1, dd->ncg_home);
+ "DD rank %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd->rank, where, i+1, dd->index_gl[i]+1, dd->ncg_home);
nerr++;
}
}
}
if (ngl != dd->ncg_tot)
{
- fprintf(stderr, "DD node %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd->rank, where, ngl, dd->ncg_tot);
+ fprintf(stderr, "DD rank %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd->rank, where, ngl, dd->ncg_tot);
nerr++;
}
{
if (have[dd->gatindex[a]] > 0)
{
- fprintf(stderr, "DD node %d: global atom %d occurs twice: index %d and %d\n", dd->rank, dd->gatindex[a]+1, have[dd->gatindex[a]], a+1);
+ fprintf(stderr, "DD rank %d: global atom %d occurs twice: index %d and %d\n", dd->rank, dd->gatindex[a]+1, have[dd->gatindex[a]], a+1);
}
else
{
{
if (a >= dd->nat_tot)
{
- fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd->rank, i+1, a+1, dd->nat_tot);
+ fprintf(stderr, "DD rank %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd->rank, i+1, a+1, dd->nat_tot);
nerr++;
}
else
have[a] = 1;
if (dd->gatindex[a] != i)
{
- fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd->rank, i+1, a+1, dd->gatindex[a]+1);
+ fprintf(stderr, "DD rank %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd->rank, i+1, a+1, dd->gatindex[a]+1);
nerr++;
}
}
if (ngl != dd->nat_tot)
{
fprintf(stderr,
- "DD node %d, %s: %d global atom indices, %d local atoms\n",
+ "DD rank %d, %s: %d global atom indices, %d local atoms\n",
dd->rank, where, ngl, dd->nat_tot);
}
for (a = 0; a < dd->nat_tot; a++)
if (have[a] == 0)
{
fprintf(stderr,
- "DD node %d, %s: local atom %d, global %d has no global index\n",
+ "DD rank %d, %s: local atom %d, global %d has no global index\n",
dd->rank, where, a+1, dd->gatindex[a]+1);
}
}
if (nerr > 0)
{
- gmx_fatal(FARGS, "DD node %d, %s: %d atom/cg index inconsistencies",
+ gmx_fatal(FARGS, "DD rank %d, %s: %d atom/cg index inconsistencies",
dd->rank, where, nerr);
}
}
/* This error should never be triggered under normal
* circumstances, but you never know ...
*/
- gmx_fatal(FARGS, "Step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with less nodes might avoid this issue.",
+ gmx_fatal(FARGS, "Step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with fewer ranks might avoid this issue.",
gmx_step_str(step, buf),
dim2char(dim), dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
}
dim2char(d), ddbox->box_size[d], ddbox->skew_fac[d],
comm->cutoff,
dd->nc[d], dd->nc[d],
- dd->nnodes > dd->nc[d] ? "cells" : "processors");
+ dd->nnodes > dd->nc[d] ? "cells" : "ranks");
if (setmode == setcellsizeslbLOCAL)
{
static void print_cg_move(FILE *fplog,
gmx_domdec_t *dd,
gmx_int64_t step, int cg, int dim, int dir,
- gmx_bool bHaveLimitdAndCMOld, real limitd,
+ gmx_bool bHaveCgcmOld, real limitd,
rvec cm_old, rvec cm_new, real pos_d)
{
gmx_domdec_comm_t *comm;
comm = dd->comm;
fprintf(fplog, "\nStep %s:\n", gmx_step_str(step, buf));
- if (bHaveLimitdAndCMOld)
+ if (limitd > 0)
{
- fprintf(fplog, "The charge group starting at atom %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
+ fprintf(fplog, "%s %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
+ dd->comm->bCGs ? "The charge group starting at atom" : "Atom",
ddglatnr(dd, dd->cgindex[cg]), limitd, dim2char(dim));
}
else
{
- fprintf(fplog, "The charge group starting at atom %d moved than the distance allowed by the domain decomposition in direction %c\n",
+ /* We don't have a limiting distance available: don't print it */
+ fprintf(fplog, "%s %d moved more than the distance allowed by the domain decomposition in direction %c\n",
+ dd->comm->bCGs ? "The charge group starting at atom" : "Atom",
ddglatnr(dd, dd->cgindex[cg]), dim2char(dim));
}
fprintf(fplog, "distance out of cell %f\n",
dir == 1 ? pos_d - comm->cell_x1[dim] : pos_d - comm->cell_x0[dim]);
- if (bHaveLimitdAndCMOld)
+ if (bHaveCgcmOld)
{
fprintf(fplog, "Old coordinates: %8.3f %8.3f %8.3f\n",
cm_old[XX], cm_old[YY], cm_old[ZZ]);
static void cg_move_error(FILE *fplog,
gmx_domdec_t *dd,
gmx_int64_t step, int cg, int dim, int dir,
- gmx_bool bHaveLimitdAndCMOld, real limitd,
+ gmx_bool bHaveCgcmOld, real limitd,
rvec cm_old, rvec cm_new, real pos_d)
{
if (fplog)
{
print_cg_move(fplog, dd, step, cg, dim, dir,
- bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ bHaveCgcmOld, limitd, cm_old, cm_new, pos_d);
}
print_cg_move(stderr, dd, step, cg, dim, dir,
- bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ bHaveCgcmOld, limitd, cm_old, cm_new, pos_d);
gmx_fatal(FARGS,
- "A charge group moved too far between two domain decomposition steps\n"
- "This usually means that your system is not well equilibrated");
+ "%s moved too far between two domain decomposition steps\n"
+ "This usually means that your system is not well equilibrated",
+ dd->comm->bCGs ? "A charge group" : "An atom");
}
static void rotate_state_atom(t_state *state, int a)
{
if (pos_d >= limit1[d])
{
- cg_move_error(fplog, dd, step, cg, d, 1, TRUE, limitd[d],
+ cg_move_error(fplog, dd, step, cg, d, 1,
+ cg_cm != state->x, limitd[d],
cg_cm[cg], cm_new, pos_d);
}
dev[d] = 1;
{
if (pos_d < limit0[d])
{
- cg_move_error(fplog, dd, step, cg, d, -1, TRUE, limitd[d],
+ cg_move_error(fplog, dd, step, cg, d, -1,
+ cg_cm != state->x, limitd[d],
cg_cm[cg], cm_new, pos_d);
}
dev[d] = -1;
{
cg_move_error(fplog, dd, step, cg, dim,
(flag & DD_FLAG_FW(d)) ? 1 : 0,
- FALSE, 0,
+ fr->cutoff_scheme == ecutsGROUP, 0,
comm->vbuf.v[buf_pos],
comm->vbuf.v[buf_pos],
comm->vbuf.v[buf_pos][dim]);
if (npme > 0 && fabs(lossp) >= DD_PERF_LOSS_WARN)
{
sprintf(buf,
- "NOTE: %.1f %% performance was lost because the PME nodes\n"
- " had %s work to do than the PP nodes.\n"
- " You might want to %s the number of PME nodes\n"
+ "NOTE: %.1f %% performance was lost because the PME ranks\n"
+ " had %s work to do than the PP ranks.\n"
+ " You might want to %s the number of PME ranks\n"
" or %s the cut-off and the grid spacing.\n",
fabs(lossp*100),
(lossp < 0) ? "less" : "more",
if (fplog)
{
fprintf(fplog,
- "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ "Domain decomposition rank %d, coordinates %d %d %d\n\n",
dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
}
if (debug)
{
fprintf(debug,
- "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ "Domain decomposition rank %d, coordinates %d %d %d\n\n",
dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
}
}
}
else if (fplog)
{
- fprintf(fplog, "#pmenodes (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr->npmenodes, dd->nc[XX], dd->nc[YY], dd->nc[XX], dd->nc[ZZ]);
+ fprintf(fplog, "Number of PME-only ranks (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr->npmenodes, dd->nc[XX], dd->nc[YY], dd->nc[XX], dd->nc[ZZ]);
fprintf(fplog,
"Will not use a Cartesian communicator for PP <-> PME\n\n");
}
if (fplog)
{
- fprintf(fplog, "Cartesian nodeid %d, coordinates %d %d %d\n\n",
+ fprintf(fplog, "Cartesian rank %d, coordinates %d %d %d\n\n",
cr->sim_nodeid, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
}
case ddnoPP_PME:
if (fplog)
{
- fprintf(fplog, "Order of the nodes: PP first, PME last\n");
+ fprintf(fplog, "Order of the ranks: PP first, PME last\n");
}
break;
case ddnoINTERLEAVE:
*/
if (fplog)
{
- fprintf(fplog, "Interleaving PP and PME nodes\n");
+ fprintf(fplog, "Interleaving PP and PME ranks\n");
}
comm->pmenodes = dd_pmenodes(cr);
break;
if (fplog)
{
- fprintf(fplog, "This is a %s only node\n\n",
+ fprintf(fplog, "This rank does only %s work.\n\n",
(cr->duty & DUTY_PP) ? "particle-particle" : "PME-mesh");
}
}
if (fplog)
{
fprintf(fplog,
- "\nInitializing Domain Decomposition on %d nodes\n", cr->nnodes);
+ "\nInitializing Domain Decomposition on %d ranks\n", cr->nnodes);
}
snew(dd, 1);
comm->cellsize_limit = 0;
comm->bBondComm = FALSE;
+ /* Atoms should be able to move by up to half the list buffer size (if > 0)
+ * within nstlist steps. Since boundaries are allowed to displace by half
+ * a cell size, DD cells should be at least the size of the list buffer.
+ */
+ comm->cellsize_limit = max(comm->cellsize_limit,
+ ir->rlistlong - max(ir->rvdw, ir->rcoulomb));
+
if (comm->bInterCGBondeds)
{
if (comm_distance_min > 0)
if (dd->nc[XX] == 0)
{
bC = (dd->bInterCGcons && rconstr > r_bonded_limit);
- sprintf(buf, "Change the number of nodes or mdrun option %s%s%s",
+ sprintf(buf, "Change the number of ranks or mdrun option %s%s%s",
!bC ? "-rdd" : "-rcon",
comm->eDLB != edlbNO ? " or -dds" : "",
bC ? " or your LINCS settings" : "");
gmx_fatal_collective(FARGS, cr, NULL,
- "There is no domain decomposition for %d nodes that is compatible with the given box and a minimum cell size of %g nm\n"
+ "There is no domain decomposition for %d ranks that is compatible with the given box and a minimum cell size of %g nm\n"
"%s\n"
"Look in the log file for details on the domain decomposition",
cr->nnodes-cr->npmenodes, limit, buf);
if (fplog)
{
fprintf(fplog,
- "Domain decomposition grid %d x %d x %d, separate PME nodes %d\n",
+ "Domain decomposition grid %d x %d x %d, separate PME ranks %d\n",
dd->nc[XX], dd->nc[YY], dd->nc[ZZ], cr->npmenodes);
}
if (cr->nnodes - dd->nnodes != cr->npmenodes)
{
gmx_fatal_collective(FARGS, cr, NULL,
- "The size of the domain decomposition grid (%d) does not match the number of nodes (%d). The total number of nodes is %d",
+ "The size of the domain decomposition grid (%d) does not match the number of ranks (%d). The total number of ranks is %d",
dd->nnodes, cr->nnodes - cr->npmenodes, cr->nnodes);
}
if (cr->npmenodes > dd->nnodes)
{
gmx_fatal_collective(FARGS, cr, NULL,
- "The number of separate PME nodes (%d) is larger than the number of PP nodes (%d), this is not supported.", cr->npmenodes, dd->nnodes);
+ "The number of separate PME ranks (%d) is larger than the number of PP ranks (%d), this is not supported.", cr->npmenodes, dd->nnodes);
}
if (cr->npmenodes > 0)
{
if (dd->pme_nodeid >= 0)
{
gmx_fatal_collective(FARGS, NULL, dd,
- "Can not have separate PME nodes without PME electrostatics");
+ "Can not have separate PME ranks without PME electrostatics");
}
}
if (vsite != NULL)
{
/* Now we have updated mdatoms, we can do the last vsite bookkeeping */
- split_vsites_over_threads(top_local->idef.il, mdatoms, FALSE, vsite);
+ split_vsites_over_threads(top_local->idef.il, top_local->idef.iparams,
+ mdatoms, FALSE, vsite);
}
if (shellfc)
if (!(cr->duty & DUTY_PME))
{
/* Send the charges and/or c6/sigmas to our PME only node */
- gmx_pme_send_parameters(cr, mdatoms->nChargePerturbed, mdatoms->nTypePerturbed,
+ gmx_pme_send_parameters(cr,
+ fr->ic,
+ mdatoms->nChargePerturbed, mdatoms->nTypePerturbed,
mdatoms->chargeA, mdatoms->chargeB,
mdatoms->sqrt_c6A, mdatoms->sqrt_c6B,
mdatoms->sigmaA, mdatoms->sigmaB,
biod.pnpi.spb.ru / alexxy / gromacs.git / blobdiff