With domain decomposition and 2 decomposition cells in a trilinic
dimension, the cut-off could be longer than the size of the
communicated domains. This could lead to some pairs close to cut-off
distance to be ignored in the force/energy calculations.
Fixes #1467
Change-Id: Id7e16d7f8fa0796d6adcf48ad6e8bbb0b88039ff
+enum { setcellsizeslbLOCAL, setcellsizeslbMASTER, setcellsizeslbPULSE_ONLY };
+
+/* Set the domain boundaries. Use for static (or no) load balancing,
+ * and also for the starting state for dynamic load balancing.
+ * setmode determine if and where the boundaries are stored, use enum above.
+ * Returns the number communication pulses in npulse.
+ */
static void set_dd_cell_sizes_slb(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
static void set_dd_cell_sizes_slb(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
- gmx_bool bMaster, ivec npulse)
+ int setmode, ivec npulse)
{
gmx_domdec_comm_t *comm;
int d, j;
{
gmx_domdec_comm_t *comm;
int d, j;
{
/* Uniform grid */
cell_dx = ddbox->box_size[d]/dd->nc[d];
{
/* Uniform grid */
cell_dx = ddbox->box_size[d]/dd->nc[d];
- if (bMaster)
- {
- for (j = 0; j < dd->nc[d]+1; j++)
- {
- dd->ma->cell_x[d][j] = ddbox->box0[d] + j*cell_dx;
- }
- }
- else
- comm->cell_x0[d] = ddbox->box0[d] + (dd->ci[d] )*cell_dx;
- comm->cell_x1[d] = ddbox->box0[d] + (dd->ci[d]+1)*cell_dx;
+ case setcellsizeslbMASTER:
+ for (j = 0; j < dd->nc[d]+1; j++)
+ {
+ dd->ma->cell_x[d][j] = ddbox->box0[d] + j*cell_dx;
+ }
+ break;
+ case setcellsizeslbLOCAL:
+ comm->cell_x0[d] = ddbox->box0[d] + (dd->ci[d] )*cell_dx;
+ comm->cell_x1[d] = ddbox->box0[d] + (dd->ci[d]+1)*cell_dx;
+ break;
+ default:
+ break;
}
cellsize = cell_dx*ddbox->skew_fac[d];
}
cellsize = cell_dx*ddbox->skew_fac[d];
- while (cellsize*npulse[d] < comm->cutoff && npulse[d] < dd->nc[d]-1)
+ while (cellsize*npulse[d] < comm->cutoff)
* all cell borders in a loop to obtain identical values
* to the master distribution case and to determine npulse.
*/
* all cell borders in a loop to obtain identical values
* to the master distribution case and to determine npulse.
*/
+ if (setmode == setcellsizeslbMASTER)
{
cell_x = dd->ma->cell_x[d];
}
{
cell_x = dd->ma->cell_x[d];
}
}
cellsize_min[d] = min(cellsize_min[d], cellsize);
}
}
cellsize_min[d] = min(cellsize_min[d], cellsize);
}
+ if (setmode == setcellsizeslbLOCAL)
{
comm->cell_x0[d] = cell_x[dd->ci[d]];
comm->cell_x1[d] = cell_x[dd->ci[d]+1];
{
comm->cell_x0[d] = cell_x[dd->ci[d]];
comm->cell_x1[d] = cell_x[dd->ci[d]+1];
+ }
+ if (setmode != setcellsizeslbMASTER)
+ {
if (d < ddbox->npbcdim &&
dd->nc[d] > 1 && npulse[d] >= dd->nc[d])
{
if (d < ddbox->npbcdim &&
dd->nc[d] > 1 && npulse[d] >= dd->nc[d])
{
- gmx_fatal_collective(FARGS, NULL, dd,
- "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
- dim2char(d), ddbox->box_size[d], ddbox->skew_fac[d],
- comm->cutoff,
- dd->nc[d], dd->nc[d],
- dd->nnodes > dd->nc[d] ? "cells" : "processors");
+ char error_string[STRLEN];
+
+ sprintf(error_string,
+ "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
+ dim2char(d), ddbox->box_size[d], ddbox->skew_fac[d],
+ comm->cutoff,
+ dd->nc[d], dd->nc[d],
+ dd->nnodes > dd->nc[d] ? "cells" : "processors");
+
+ if (setmode == setcellsizeslbLOCAL)
+ {
+ gmx_fatal_collective(FARGS, NULL, dd, error_string);
+ }
+ else
+ {
+ gmx_fatal(FARGS, error_string);
+ }
- set_dd_cell_sizes_slb(dd, ddbox, FALSE, npulse);
+ set_dd_cell_sizes_slb(dd, ddbox, setcellsizeslbLOCAL, npulse);
realloc_comm_ind(dd, npulse);
}
realloc_comm_ind(dd, npulse);
}
int i, cg_gl;
int *ibuf, buf2[2] = { 0, 0 };
gmx_bool bMaster = DDMASTER(dd);
int i, cg_gl;
int *ibuf, buf2[2] = { 0, 0 };
gmx_bool bMaster = DDMASTER(dd);
if (bMaster)
{
ma = dd->ma;
if (bMaster)
{
ma = dd->ma;
- set_dd_cell_sizes_slb(dd, ddbox, TRUE, npulse);
+ set_dd_cell_sizes_slb(dd, ddbox, setcellsizeslbMASTER, npulse);
distribute_cg(fplog, step, box, ddbox->tric_dir, cgs, pos, dd);
for (i = 0; i < dd->nnodes; i++)
distribute_cg(fplog, step, box, ddbox->tric_dir, cgs, pos, dd);
for (i = 0; i < dd->nnodes; i++)
- set_dd_cell_sizes_slb(dd, ddbox, FALSE, np);
+ set_dd_cell_sizes_slb(dd, ddbox, setcellsizeslbPULSE_ONLY, np);
fprintf(fplog, "The initial number of communication pulses is:");
for (d = 0; d < dd->ndim; d++)
{
fprintf(fplog, "The initial number of communication pulses is:");
for (d = 0; d < dd->ndim; d++)
{
{
bt[i] = ddbox->box_size[i]*ddbox->skew_fac[i];
{
bt[i] = ddbox->box_size[i]*ddbox->skew_fac[i];
- /* Without PBC there are no cell size limits with 2 cells */
+ /* Without PBC and with 2 cells, there are no lower limits on the cell size */
if (!(i >= ddbox->npbcdim && nc[i] <= 2) && bt[i] < nc[i]*limit)
{
return -1;
}
if (!(i >= ddbox->npbcdim && nc[i] <= 2) && bt[i] < nc[i]*limit)
{
return -1;
}
+ /* With PBC, check if the cut-off fits in nc[i]-1 cells */
+ if (i < ddbox->npbcdim && nc[i] > 1 && (nc[i] - 1)*bt[i] < nc[i]*cutoff)
+ {
+ return -1;
+ }