src/mdlib/domdec.c

   1 /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
   2  *
   3  *
   4  * This file is part of Gromacs        Copyright (c) 1991-2008
   5  * David van der Spoel, Erik Lindahl, Berk Hess, University of Groningen.
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * To help us fund GROMACS development, we humbly ask that you cite
  13  * the research papers on the package. Check out http://www.gromacs.org
  14  *
  15  * And Hey:
  16  * Gnomes, ROck Monsters And Chili Sauce
  17  */
  18
  19 #ifdef HAVE_CONFIG_H
  20 #include <config.h>
  21 #endif
  22
  23 #include <stdio.h>
  24 #include <time.h>
  25 #include <math.h>
  26 #include <string.h>
  27 #include <stdlib.h>
  28 #include "typedefs.h"
  29 #include "smalloc.h"
  30 #include "gmx_fatal.h"
  31 #include "gmx_fatal_collective.h"
  32 #include "vec.h"
  33 #include "domdec.h"
  34 #include "domdec_network.h"
  35 #include "nrnb.h"
  36 #include "pbc.h"
  37 #include "chargegroup.h"
  38 #include "constr.h"
  39 #include "mdatoms.h"
  40 #include "names.h"
  41 #include "pdbio.h"
  42 #include "futil.h"
  43 #include "force.h"
  44 #include "pme.h"
  45 #include "pull.h"
  46 #include "pull_rotation.h"
  47 #include "gmx_wallcycle.h"
  48 #include "mdrun.h"
  49 #include "nsgrid.h"
  50 #include "shellfc.h"
  51 #include "mtop_util.h"
  52 #include "gmxfio.h"
  53 #include "gmx_ga2la.h"
  54 #include "gmx_sort.h"
  55 #include "nbnxn_search.h"
  56 #include "bondf.h"
  57 #include "gmx_omp_nthreads.h"
  58
  59 #ifdef GMX_LIB_MPI
  60 #include <mpi.h>
  61 #endif
  62 #ifdef GMX_THREAD_MPI
  63 #include "tmpi.h"
  64 #endif
  65
  66 #define DDRANK(dd,rank)    (rank)
  67 #define DDMASTERRANK(dd)   (dd->masterrank)
  68
  69 typedef struct gmx_domdec_master
  70 {
  71     /* The cell boundaries */
  72     real **cell_x;
  73     /* The global charge group division */
  74     int  *ncg;     /* Number of home charge groups for each node */
  75     int  *index;   /* Index of nnodes+1 into cg */
  76     int  *cg;      /* Global charge group index */
  77     int  *nat;     /* Number of home atoms for each node. */
  78     int  *ibuf;    /* Buffer for communication */
  79     rvec *vbuf;    /* Buffer for state scattering and gathering */
  80 } gmx_domdec_master_t;
  81
  82 typedef struct
  83 {
  84     /* The numbers of charge groups to send and receive for each cell
  85      * that requires communication, the last entry contains the total
  86      * number of atoms that needs to be communicated.
  87      */
  88     int nsend[DD_MAXIZONE+2];
  89     int nrecv[DD_MAXIZONE+2];
  90     /* The charge groups to send */
  91     int *index;
  92     int nalloc;
  93     /* The atom range for non-in-place communication */
  94     int cell2at0[DD_MAXIZONE];
  95     int cell2at1[DD_MAXIZONE];
  96 } gmx_domdec_ind_t;
  97
  98 typedef struct
  99 {
 100     int  np;                   /* Number of grid pulses in this dimension */
 101     int  np_dlb;               /* For dlb, for use with edlbAUTO          */
 102     gmx_domdec_ind_t *ind;     /* The indices to communicate, size np     */
 103     int  np_nalloc;
 104     gmx_bool bInPlace;             /* Can we communicate in place?            */
 105 } gmx_domdec_comm_dim_t;
 106
 107 typedef struct
 108 {
 109     gmx_bool *bCellMin;    /* Temp. var.: is this cell size at the limit     */
 110     real *cell_f;      /* State var.: cell boundaries, box relative      */
 111     real *old_cell_f;  /* Temp. var.: old cell size                      */
 112     real *cell_f_max0; /* State var.: max lower boundary, incl neighbors */
 113     real *cell_f_min1; /* State var.: min upper boundary, incl neighbors */
 114     real *bound_min;   /* Temp. var.: lower limit for cell boundary      */
 115     real *bound_max;   /* Temp. var.: upper limit for cell boundary      */
 116     gmx_bool bLimited;     /* State var.: is DLB limited in this dim and row */
 117     real *buf_ncd;     /* Temp. var.                                     */
 118 } gmx_domdec_root_t;
 119
 120 #define DD_NLOAD_MAX 9
 121
 122 /* Here floats are accurate enough, since these variables
 123  * only influence the load balancing, not the actual MD results.
 124  */
 125 typedef struct
 126 {
 127     int  nload;
 128     float *load;
 129     float sum;
 130     float max;
 131     float sum_m;
 132     float cvol_min;
 133     float mdf;
 134     float pme;
 135     int   flags;
 136 } gmx_domdec_load_t;
 137
 138 typedef struct
 139 {
 140     int  nsc;
 141     int  ind_gl;
 142     int  ind;
 143 } gmx_cgsort_t;
 144
 145 typedef struct
 146 {
 147     gmx_cgsort_t *sort;
 148     gmx_cgsort_t *sort2;
 149     int  sort_nalloc;
 150     gmx_cgsort_t *sort_new;
 151     int  sort_new_nalloc;
 152     int  *ibuf;
 153     int  ibuf_nalloc;
 154 } gmx_domdec_sort_t;
 155
 156 typedef struct
 157 {
 158     rvec *v;
 159     int  nalloc;
 160 } vec_rvec_t;
 161
 162 /* This enum determines the order of the coordinates.
 163  * ddnatHOME and ddnatZONE should be first and second,
 164  * the others can be ordered as wanted.
 165  */
 166 enum { ddnatHOME, ddnatZONE, ddnatVSITE, ddnatCON, ddnatNR };
 167
 168 enum { edlbAUTO, edlbNO, edlbYES, edlbNR };
 169 const char *edlb_names[edlbNR] = { "auto", "no", "yes" };
 170
 171 typedef struct
 172 {
 173     int  dim;      /* The dimension                                          */
 174     gmx_bool dim_match;/* Tells if DD and PME dims match                         */
 175     int  nslab;    /* The number of PME slabs in this dimension              */
 176     real *slb_dim_f; /* Cell sizes for determining the PME comm. with SLB    */
 177     int  *pp_min;  /* The minimum pp node location, size nslab               */
 178     int  *pp_max;  /* The maximum pp node location,size nslab                */
 179     int  maxshift; /* The maximum shift for coordinate redistribution in PME */
 180 } gmx_ddpme_t;
 181
 182 typedef struct
 183 {
 184     real min0;    /* The minimum bottom of this zone                        */
 185     real max1;    /* The maximum top of this zone                           */
 186     real min1;    /* The minimum top of this zone                           */
 187     real mch0;    /* The maximum bottom communicaton height for this zone   */
 188     real mch1;    /* The maximum top communicaton height for this zone      */
 189     real p1_0;    /* The bottom value of the first cell in this zone        */
 190     real p1_1;    /* The top value of the first cell in this zone           */
 191 } gmx_ddzone_t;
 192
 193 typedef struct
 194 {
 195     gmx_domdec_ind_t ind;
 196     int *ibuf;
 197     int ibuf_nalloc;
 198     vec_rvec_t vbuf;
 199     int nsend;
 200     int nat;
 201     int nsend_zone;
 202 } dd_comm_setup_work_t;
 203
 204 typedef struct gmx_domdec_comm
 205 {
 206     /* All arrays are indexed with 0 to dd->ndim (not Cartesian indexing),
 207      * unless stated otherwise.
 208      */
 209
 210     /* The number of decomposition dimensions for PME, 0: no PME */
 211     int  npmedecompdim;
 212     /* The number of nodes doing PME (PP/PME or only PME) */
 213     int  npmenodes;
 214     int  npmenodes_x;
 215     int  npmenodes_y;
 216     /* The communication setup including the PME only nodes */
 217     gmx_bool bCartesianPP_PME;
 218     ivec ntot;
 219     int  cartpmedim;
 220     int  *pmenodes;          /* size npmenodes                         */
 221     int  *ddindex2simnodeid; /* size npmenodes, only with bCartesianPP
 222                               * but with bCartesianPP_PME              */
 223     gmx_ddpme_t ddpme[2];
 224
 225     /* The DD particle-particle nodes only */
 226     gmx_bool bCartesianPP;
 227     int  *ddindex2ddnodeid; /* size npmenode, only with bCartesianPP_PME */
 228
 229     /* The global charge groups */
 230     t_block cgs_gl;
 231
 232     /* Should we sort the cgs */
 233     int  nstSortCG;
 234     gmx_domdec_sort_t *sort;
 235
 236     /* Are there charge groups? */
 237     gmx_bool bCGs;
 238
 239     /* Are there bonded and multi-body interactions between charge groups? */
 240     gmx_bool bInterCGBondeds;
 241     gmx_bool bInterCGMultiBody;
 242
 243     /* Data for the optional bonded interaction atom communication range */
 244     gmx_bool bBondComm;
 245     t_blocka *cglink;
 246     char *bLocalCG;
 247
 248     /* The DLB option */
 249     int  eDLB;
 250     /* Are we actually using DLB? */
 251     gmx_bool bDynLoadBal;
 252
 253     /* Cell sizes for static load balancing, first index cartesian */
 254     real **slb_frac;
 255
 256     /* The width of the communicated boundaries */
 257     real cutoff_mbody;
 258     real cutoff;
 259     /* The minimum cell size (including triclinic correction) */
 260     rvec cellsize_min;
 261     /* For dlb, for use with edlbAUTO */
 262     rvec cellsize_min_dlb;
 263     /* The lower limit for the DD cell size with DLB */
 264     real cellsize_limit;
 265     /* Effectively no NB cut-off limit with DLB for systems without PBC? */
 266     gmx_bool bVacDLBNoLimit;
 267
 268     /* tric_dir is only stored here because dd_get_ns_ranges needs it */
 269     ivec tric_dir;
 270     /* box0 and box_size are required with dim's without pbc and -gcom */
 271     rvec box0;
 272     rvec box_size;
 273
 274     /* The cell boundaries */
 275     rvec cell_x0;
 276     rvec cell_x1;
 277
 278     /* The old location of the cell boundaries, to check cg displacements */
 279     rvec old_cell_x0;
 280     rvec old_cell_x1;
 281
 282     /* The communication setup and charge group boundaries for the zones */
 283     gmx_domdec_zones_t zones;
 284
 285     /* The zone limits for DD dimensions 1 and 2 (not 0), determined from
 286      * cell boundaries of neighboring cells for dynamic load balancing.
 287      */
 288     gmx_ddzone_t zone_d1[2];
 289     gmx_ddzone_t zone_d2[2][2];
 290
 291     /* The coordinate/force communication setup and indices */
 292     gmx_domdec_comm_dim_t cd[DIM];
 293     /* The maximum number of cells to communicate with in one dimension */
 294     int  maxpulse;
 295
 296     /* Which cg distribution is stored on the master node */
 297     int master_cg_ddp_count;
 298
 299     /* The number of cg's received from the direct neighbors */
 300     int  zone_ncg1[DD_MAXZONE];
 301
 302     /* The atom counts, the range for each type t is nat[t-1] <= at < nat[t] */
 303     int  nat[ddnatNR];
 304
 305     /* Array for signalling if atoms have moved to another domain */
 306     int  *moved;
 307     int  moved_nalloc;
 308
 309     /* Communication buffer for general use */
 310     int  *buf_int;
 311     int  nalloc_int;
 312
 313     /* Communication buffer for general use */
 314     vec_rvec_t vbuf;
 315
 316     /* Temporary storage for thread parallel communication setup */
 317     int nth;
 318     dd_comm_setup_work_t *dth;
 319
 320     /* Communication buffers only used with multiple grid pulses */
 321     int  *buf_int2;
 322     int  nalloc_int2;
 323     vec_rvec_t vbuf2;
 324
 325     /* Communication buffers for local redistribution */
 326     int  **cggl_flag;
 327     int  cggl_flag_nalloc[DIM*2];
 328     rvec **cgcm_state;
 329     int  cgcm_state_nalloc[DIM*2];
 330
 331     /* Cell sizes for dynamic load balancing */
 332     gmx_domdec_root_t **root;
 333     real *cell_f_row;
 334     real cell_f0[DIM];
 335     real cell_f1[DIM];
 336     real cell_f_max0[DIM];
 337     real cell_f_min1[DIM];
 338
 339     /* Stuff for load communication */
 340     gmx_bool bRecordLoad;
 341     gmx_domdec_load_t *load;
 342 #ifdef GMX_MPI
 343     MPI_Comm *mpi_comm_load;
 344 #endif
 345
 346     /* Maximum DLB scaling per load balancing step in percent */
 347     int dlb_scale_lim;
 348
 349     /* Cycle counters */
 350     float cycl[ddCyclNr];
 351     int   cycl_n[ddCyclNr];
 352     float cycl_max[ddCyclNr];
 353     /* Flop counter (0=no,1=yes,2=with (eFlop-1)*5% noise */
 354     int eFlop;
 355     double flop;
 356     int    flop_n;
 357     /* Have often have did we have load measurements */
 358     int    n_load_have;
 359     /* Have often have we collected the load measurements */
 360     int    n_load_collect;
 361
 362     /* Statistics */
 363     double sum_nat[ddnatNR-ddnatZONE];
 364     int    ndecomp;
 365     int    nload;
 366     double load_step;
 367     double load_sum;
 368     double load_max;
 369     ivec   load_lim;
 370     double load_mdf;
 371     double load_pme;
 372
 373     /* The last partition step */
 374     gmx_large_int_t partition_step;
 375
 376     /* Debugging */
 377     int  nstDDDump;
 378     int  nstDDDumpGrid;
 379     int  DD_debug;
 380 } gmx_domdec_comm_t;
 381
 382 /* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */
 383 #define DD_CGIBS 2
 384
 385 /* The flags for the cggl_flag buffer in gmx_domdec_comm_t */
 386 #define DD_FLAG_NRCG  65535
 387 #define DD_FLAG_FW(d) (1<<(16+(d)*2))
 388 #define DD_FLAG_BW(d) (1<<(16+(d)*2+1))
 389
 390 /* Zone permutation required to obtain consecutive charge groups
 391  * for neighbor searching.
 392  */
 393 static const int zone_perm[3][4] = { {0,0,0,0},{1,0,0,0},{3,0,1,2} };
 394
 395 /* dd_zo and dd_zp3/dd_zp2 are set up such that i zones with non-zero
 396  * components see only j zones with that component 0.
 397  */
 398
 399 /* The DD zone order */
 400 static const ivec dd_zo[DD_MAXZONE] =
 401   {{0,0,0},{1,0,0},{1,1,0},{0,1,0},{0,1,1},{0,0,1},{1,0,1},{1,1,1}};
 402
 403 /* The 3D setup */
 404 #define dd_z3n  8
 405 #define dd_zp3n 4
 406 static const ivec dd_zp3[dd_zp3n] = {{0,0,8},{1,3,6},{2,5,6},{3,5,7}};
 407
 408 /* The 2D setup */
 409 #define dd_z2n  4
 410 #define dd_zp2n 2
 411 static const ivec dd_zp2[dd_zp2n] = {{0,0,4},{1,3,4}};
 412
 413 /* The 1D setup */
 414 #define dd_z1n  2
 415 #define dd_zp1n 1
 416 static const ivec dd_zp1[dd_zp1n] = {{0,0,2}};
 417
 418 /* Factors used to avoid problems due to rounding issues */
 419 #define DD_CELL_MARGIN       1.0001
 420 #define DD_CELL_MARGIN2      1.00005
 421 /* Factor to account for pressure scaling during nstlist steps */
 422 #define DD_PRES_SCALE_MARGIN 1.02
 423
 424 /* Allowed performance loss before we DLB or warn */
 425 #define DD_PERF_LOSS 0.05
 426
 427 #define DD_CELL_F_SIZE(dd,di) ((dd)->nc[(dd)->dim[(di)]]+1+(di)*2+1+(di))
 428
 429 /* Use separate MPI send and receive commands
 430  * when nnodes <= GMX_DD_NNODES_SENDRECV.
 431  * This saves memory (and some copying for small nnodes).
 432  * For high parallelization scatter and gather calls are used.
 433  */
 434 #define GMX_DD_NNODES_SENDRECV 4
 435
 436
 437 /*
 438 #define dd_index(n,i) ((((i)[ZZ]*(n)[YY] + (i)[YY])*(n)[XX]) + (i)[XX])
 439
 440 static void index2xyz(ivec nc,int ind,ivec xyz)
 441 {
 442   xyz[XX] = ind % nc[XX];
 443   xyz[YY] = (ind / nc[XX]) % nc[YY];
 444   xyz[ZZ] = ind / (nc[YY]*nc[XX]);
 445 }
 446 */
 447
 448 /* This order is required to minimize the coordinate communication in PME
 449  * which uses decomposition in the x direction.
 450  */
 451 #define dd_index(n,i) ((((i)[XX]*(n)[YY] + (i)[YY])*(n)[ZZ]) + (i)[ZZ])
 452
 453 static void ddindex2xyz(ivec nc,int ind,ivec xyz)
 454 {
 455     xyz[XX] = ind / (nc[YY]*nc[ZZ]);
 456     xyz[YY] = (ind / nc[ZZ]) % nc[YY];
 457     xyz[ZZ] = ind % nc[ZZ];
 458 }
 459
 460 static int ddcoord2ddnodeid(gmx_domdec_t *dd,ivec c)
 461 {
 462     int ddindex;
 463     int ddnodeid=-1;
 464
 465     ddindex = dd_index(dd->nc,c);
 466     if (dd->comm->bCartesianPP_PME)
 467     {
 468         ddnodeid = dd->comm->ddindex2ddnodeid[ddindex];
 469     }
 470     else if (dd->comm->bCartesianPP)
 471     {
 472 #ifdef GMX_MPI
 473         MPI_Cart_rank(dd->mpi_comm_all,c,&ddnodeid);
 474 #endif
 475     }
 476     else
 477     {
 478         ddnodeid = ddindex;
 479     }
 480
 481     return ddnodeid;
 482 }
 483
 484 static gmx_bool dynamic_dd_box(gmx_ddbox_t *ddbox,t_inputrec *ir)
 485 {
 486     return (ddbox->nboundeddim < DIM || DYNAMIC_BOX(*ir));
 487 }
 488
 489 int ddglatnr(gmx_domdec_t *dd,int i)
 490 {
 491     int atnr;
 492
 493     if (dd == NULL)
 494     {
 495         atnr = i + 1;
 496     }
 497     else
 498     {
 499         if (i >= dd->comm->nat[ddnatNR-1])
 500         {
 501             gmx_fatal(FARGS,"glatnr called with %d, which is larger than the local number of atoms (%d)",i,dd->comm->nat[ddnatNR-1]);
 502         }
 503         atnr = dd->gatindex[i] + 1;
 504     }
 505
 506     return atnr;
 507 }
 508
 509 t_block *dd_charge_groups_global(gmx_domdec_t *dd)
 510 {
 511     return &dd->comm->cgs_gl;
 512 }
 513
 514 static void vec_rvec_init(vec_rvec_t *v)
 515 {
 516     v->nalloc = 0;
 517     v->v      = NULL;
 518 }
 519
 520 static void vec_rvec_check_alloc(vec_rvec_t *v,int n)
 521 {
 522     if (n > v->nalloc)
 523     {
 524         v->nalloc = over_alloc_dd(n);
 525         srenew(v->v,v->nalloc);
 526     }
 527 }
 528
 529 void dd_store_state(gmx_domdec_t *dd,t_state *state)
 530 {
 531     int i;
 532
 533     if (state->ddp_count != dd->ddp_count)
 534     {
 535         gmx_incons("The state does not the domain decomposition state");
 536     }
 537
 538     state->ncg_gl = dd->ncg_home;
 539     if (state->ncg_gl > state->cg_gl_nalloc)
 540     {
 541         state->cg_gl_nalloc = over_alloc_dd(state->ncg_gl);
 542         srenew(state->cg_gl,state->cg_gl_nalloc);
 543     }
 544     for(i=0; i<state->ncg_gl; i++)
 545     {
 546         state->cg_gl[i] = dd->index_gl[i];
 547     }
 548
 549     state->ddp_count_cg_gl = dd->ddp_count;
 550 }
 551
 552 gmx_domdec_zones_t *domdec_zones(gmx_domdec_t *dd)
 553 {
 554     return &dd->comm->zones;
 555 }
 556
 557 void dd_get_ns_ranges(gmx_domdec_t *dd,int icg,
 558                       int *jcg0,int *jcg1,ivec shift0,ivec shift1)
 559 {
 560     gmx_domdec_zones_t *zones;
 561     int izone,d,dim;
 562
 563     zones = &dd->comm->zones;
 564
 565     izone = 0;
 566     while (icg >= zones->izone[izone].cg1)
 567     {
 568         izone++;
 569     }
 570
 571     if (izone == 0)
 572     {
 573         *jcg0 = icg;
 574     }
 575     else if (izone < zones->nizone)
 576     {
 577         *jcg0 = zones->izone[izone].jcg0;
 578     }
 579     else
 580     {
 581         gmx_fatal(FARGS,"DD icg %d out of range: izone (%d) >= nizone (%d)",
 582                   icg,izone,zones->nizone);
 583     }
 584
 585     *jcg1 = zones->izone[izone].jcg1;
 586
 587     for(d=0; d<dd->ndim; d++)
 588     {
 589         dim = dd->dim[d];
 590         shift0[dim] = zones->izone[izone].shift0[dim];
 591         shift1[dim] = zones->izone[izone].shift1[dim];
 592         if (dd->comm->tric_dir[dim] || (dd->bGridJump && d > 0))
 593         {
 594             /* A conservative approach, this can be optimized */
 595             shift0[dim] -= 1;
 596             shift1[dim] += 1;
 597         }
 598     }
 599 }
 600
 601 int dd_natoms_vsite(gmx_domdec_t *dd)
 602 {
 603     return dd->comm->nat[ddnatVSITE];
 604 }
 605
 606 void dd_get_constraint_range(gmx_domdec_t *dd,int *at_start,int *at_end)
 607 {
 608     *at_start = dd->comm->nat[ddnatCON-1];
 609     *at_end   = dd->comm->nat[ddnatCON];
 610 }
 611
 612 void dd_move_x(gmx_domdec_t *dd,matrix box,rvec x[])
 613 {
 614     int  nzone,nat_tot,n,d,p,i,j,at0,at1,zone;
 615     int  *index,*cgindex;
 616     gmx_domdec_comm_t *comm;
 617     gmx_domdec_comm_dim_t *cd;
 618     gmx_domdec_ind_t *ind;
 619     rvec shift={0,0,0},*buf,*rbuf;
 620     gmx_bool bPBC,bScrew;
 621
 622     comm = dd->comm;
 623
 624     cgindex = dd->cgindex;
 625
 626     buf = comm->vbuf.v;
 627
 628     nzone = 1;
 629     nat_tot = dd->nat_home;
 630     for(d=0; d<dd->ndim; d++)
 631     {
 632         bPBC   = (dd->ci[dd->dim[d]] == 0);
 633         bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
 634         if (bPBC)
 635         {
 636             copy_rvec(box[dd->dim[d]],shift);
 637         }
 638         cd = &comm->cd[d];
 639         for(p=0; p<cd->np; p++)
 640         {
 641             ind = &cd->ind[p];
 642             index = ind->index;
 643             n = 0;
 644             if (!bPBC)
 645             {
 646                 for(i=0; i<ind->nsend[nzone]; i++)
 647                 {
 648                     at0 = cgindex[index[i]];
 649                     at1 = cgindex[index[i]+1];
 650                     for(j=at0; j<at1; j++)
 651                     {
 652                         copy_rvec(x[j],buf[n]);
 653                         n++;
 654                     }
 655                 }
 656             }
 657             else if (!bScrew)
 658             {
 659                 for(i=0; i<ind->nsend[nzone]; i++)
 660                 {
 661                     at0 = cgindex[index[i]];
 662                     at1 = cgindex[index[i]+1];
 663                     for(j=at0; j<at1; j++)
 664                     {
 665                         /* We need to shift the coordinates */
 666                         rvec_add(x[j],shift,buf[n]);
 667                         n++;
 668                     }
 669                 }
 670             }
 671             else
 672             {
 673                 for(i=0; i<ind->nsend[nzone]; i++)
 674                 {
 675                     at0 = cgindex[index[i]];
 676                     at1 = cgindex[index[i]+1];
 677                     for(j=at0; j<at1; j++)
 678                     {
 679                         /* Shift x */
 680                         buf[n][XX] = x[j][XX] + shift[XX];
 681                         /* Rotate y and z.
 682                          * This operation requires a special shift force
 683                          * treatment, which is performed in calc_vir.
 684                          */
 685                         buf[n][YY] = box[YY][YY] - x[j][YY];
 686                         buf[n][ZZ] = box[ZZ][ZZ] - x[j][ZZ];
 687                         n++;
 688                     }
 689                 }
 690             }
 691
 692             if (cd->bInPlace)
 693             {
 694                 rbuf = x + nat_tot;
 695             }
 696             else
 697             {
 698                 rbuf = comm->vbuf2.v;
 699             }
 700             /* Send and receive the coordinates */
 701             dd_sendrecv_rvec(dd, d, dddirBackward,
 702                              buf,  ind->nsend[nzone+1],
 703                              rbuf, ind->nrecv[nzone+1]);
 704             if (!cd->bInPlace)
 705             {
 706                 j = 0;
 707                 for(zone=0; zone<nzone; zone++)
 708                 {
 709                     for(i=ind->cell2at0[zone]; i<ind->cell2at1[zone]; i++)
 710                     {
 711                         copy_rvec(rbuf[j],x[i]);
 712                         j++;
 713                     }
 714                 }
 715             }
 716             nat_tot += ind->nrecv[nzone+1];
 717         }
 718         nzone += nzone;
 719     }
 720 }
 721
 722 void dd_move_f(gmx_domdec_t *dd,rvec f[],rvec *fshift)
 723 {
 724     int  nzone,nat_tot,n,d,p,i,j,at0,at1,zone;
 725     int  *index,*cgindex;
 726     gmx_domdec_comm_t *comm;
 727     gmx_domdec_comm_dim_t *cd;
 728     gmx_domdec_ind_t *ind;
 729     rvec *buf,*sbuf;
 730     ivec vis;
 731     int  is;
 732     gmx_bool bPBC,bScrew;
 733
 734     comm = dd->comm;
 735
 736     cgindex = dd->cgindex;
 737
 738     buf = comm->vbuf.v;
 739
 740     n = 0;
 741     nzone = comm->zones.n/2;
 742     nat_tot = dd->nat_tot;
 743     for(d=dd->ndim-1; d>=0; d--)
 744     {
 745         bPBC   = (dd->ci[dd->dim[d]] == 0);
 746         bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
 747         if (fshift == NULL && !bScrew)
 748         {
 749             bPBC = FALSE;
 750         }
 751         /* Determine which shift vector we need */
 752         clear_ivec(vis);
 753         vis[dd->dim[d]] = 1;
 754         is = IVEC2IS(vis);
 755
 756         cd = &comm->cd[d];
 757         for(p=cd->np-1; p>=0; p--) {
 758             ind = &cd->ind[p];
 759             nat_tot -= ind->nrecv[nzone+1];
 760             if (cd->bInPlace)
 761             {
 762                 sbuf = f + nat_tot;
 763             }
 764             else
 765             {
 766                 sbuf = comm->vbuf2.v;
 767                 j = 0;
 768                 for(zone=0; zone<nzone; zone++)
 769                 {
 770                     for(i=ind->cell2at0[zone]; i<ind->cell2at1[zone]; i++)
 771                     {
 772                         copy_rvec(f[i],sbuf[j]);
 773                         j++;
 774                     }
 775                 }
 776             }
 777             /* Communicate the forces */
 778             dd_sendrecv_rvec(dd, d, dddirForward,
 779                              sbuf, ind->nrecv[nzone+1],
 780                              buf,  ind->nsend[nzone+1]);
 781             index = ind->index;
 782             /* Add the received forces */
 783             n = 0;
 784             if (!bPBC)
 785             {
 786                 for(i=0; i<ind->nsend[nzone]; i++)
 787                 {
 788                     at0 = cgindex[index[i]];
 789                     at1 = cgindex[index[i]+1];
 790                     for(j=at0; j<at1; j++)
 791                     {
 792                         rvec_inc(f[j],buf[n]);
 793                         n++;
 794                     }
 795                 }
 796             }
 797             else if (!bScrew)
 798             {
 799                 for(i=0; i<ind->nsend[nzone]; i++)
 800                 {
 801                     at0 = cgindex[index[i]];
 802                     at1 = cgindex[index[i]+1];
 803                     for(j=at0; j<at1; j++)
 804                     {
 805                         rvec_inc(f[j],buf[n]);
 806                         /* Add this force to the shift force */
 807                         rvec_inc(fshift[is],buf[n]);
 808                         n++;
 809                     }
 810                 }
 811             }
 812             else
 813             {
 814                 for(i=0; i<ind->nsend[nzone]; i++)
 815                 {
 816                     at0 = cgindex[index[i]];
 817                     at1 = cgindex[index[i]+1];
 818                     for(j=at0; j<at1; j++)
 819                     {
 820                         /* Rotate the force */
 821                         f[j][XX] += buf[n][XX];
 822                         f[j][YY] -= buf[n][YY];
 823                         f[j][ZZ] -= buf[n][ZZ];
 824                         if (fshift)
 825                         {
 826                             /* Add this force to the shift force */
 827                             rvec_inc(fshift[is],buf[n]);
 828                         }
 829                         n++;
 830                     }
 831                 }
 832             }
 833         }
 834         nzone /= 2;
 835     }
 836 }
 837
 838 void dd_atom_spread_real(gmx_domdec_t *dd,real v[])
 839 {
 840     int  nzone,nat_tot,n,d,p,i,j,at0,at1,zone;
 841     int  *index,*cgindex;
 842     gmx_domdec_comm_t *comm;
 843     gmx_domdec_comm_dim_t *cd;
 844     gmx_domdec_ind_t *ind;
 845     real *buf,*rbuf;
 846
 847     comm = dd->comm;
 848
 849     cgindex = dd->cgindex;
 850
 851     buf = &comm->vbuf.v[0][0];
 852
 853     nzone = 1;
 854     nat_tot = dd->nat_home;
 855     for(d=0; d<dd->ndim; d++)
 856     {
 857         cd = &comm->cd[d];
 858         for(p=0; p<cd->np; p++)
 859         {
 860             ind = &cd->ind[p];
 861             index = ind->index;
 862             n = 0;
 863             for(i=0; i<ind->nsend[nzone]; i++)
 864             {
 865                 at0 = cgindex[index[i]];
 866                 at1 = cgindex[index[i]+1];
 867                 for(j=at0; j<at1; j++)
 868                 {
 869                     buf[n] = v[j];
 870                     n++;
 871                 }
 872             }
 873
 874             if (cd->bInPlace)
 875             {
 876                 rbuf = v + nat_tot;
 877             }
 878             else
 879             {
 880                 rbuf = &comm->vbuf2.v[0][0];
 881             }
 882             /* Send and receive the coordinates */
 883             dd_sendrecv_real(dd, d, dddirBackward,
 884                              buf,  ind->nsend[nzone+1],
 885                              rbuf, ind->nrecv[nzone+1]);
 886             if (!cd->bInPlace)
 887             {
 888                 j = 0;
 889                 for(zone=0; zone<nzone; zone++)
 890                 {
 891                     for(i=ind->cell2at0[zone]; i<ind->cell2at1[zone]; i++)
 892                     {
 893                         v[i] = rbuf[j];
 894                         j++;
 895                     }
 896                 }
 897             }
 898             nat_tot += ind->nrecv[nzone+1];
 899         }
 900         nzone += nzone;
 901     }
 902 }
 903
 904 void dd_atom_sum_real(gmx_domdec_t *dd,real v[])
 905 {
 906     int  nzone,nat_tot,n,d,p,i,j,at0,at1,zone;
 907     int  *index,*cgindex;
 908     gmx_domdec_comm_t *comm;
 909     gmx_domdec_comm_dim_t *cd;
 910     gmx_domdec_ind_t *ind;
 911     real *buf,*sbuf;
 912
 913     comm = dd->comm;
 914
 915     cgindex = dd->cgindex;
 916
 917     buf = &comm->vbuf.v[0][0];
 918
 919     n = 0;
 920     nzone = comm->zones.n/2;
 921     nat_tot = dd->nat_tot;
 922     for(d=dd->ndim-1; d>=0; d--)
 923     {
 924         cd = &comm->cd[d];
 925         for(p=cd->np-1; p>=0; p--) {
 926             ind = &cd->ind[p];
 927             nat_tot -= ind->nrecv[nzone+1];
 928             if (cd->bInPlace)
 929             {
 930                 sbuf = v + nat_tot;
 931             }
 932             else
 933             {
 934                 sbuf = &comm->vbuf2.v[0][0];
 935                 j = 0;
 936                 for(zone=0; zone<nzone; zone++)
 937                 {
 938                     for(i=ind->cell2at0[zone]; i<ind->cell2at1[zone]; i++)
 939                     {
 940                         sbuf[j] = v[i];
 941                         j++;
 942                     }
 943                 }
 944             }
 945             /* Communicate the forces */
 946             dd_sendrecv_real(dd, d, dddirForward,
 947                              sbuf, ind->nrecv[nzone+1],
 948                              buf,  ind->nsend[nzone+1]);
 949             index = ind->index;
 950             /* Add the received forces */
 951             n = 0;
 952             for(i=0; i<ind->nsend[nzone]; i++)
 953             {
 954                 at0 = cgindex[index[i]];
 955                 at1 = cgindex[index[i]+1];
 956                 for(j=at0; j<at1; j++)
 957                 {
 958                     v[j] += buf[n];
 959                     n++;
 960                 }
 961             }
 962         }
 963         nzone /= 2;
 964     }
 965 }
 966
 967 static void print_ddzone(FILE *fp,int d,int i,int j,gmx_ddzone_t *zone)
 968 {
 969     fprintf(fp,"zone d0 %d d1 %d d2 %d  min0 %6.3f max1 %6.3f mch0 %6.3f mch1 %6.3f p1_0 %6.3f p1_1 %6.3f\n",
 970             d,i,j,
 971             zone->min0,zone->max1,
 972             zone->mch0,zone->mch0,
 973             zone->p1_0,zone->p1_1);
 974 }
 975
 976
 977 #define DDZONECOMM_MAXZONE  5
 978 #define DDZONECOMM_BUFSIZE  3
 979
 980 static void dd_sendrecv_ddzone(const gmx_domdec_t *dd,
 981                                int ddimind,int direction,
 982                                gmx_ddzone_t *buf_s,int n_s,
 983                                gmx_ddzone_t *buf_r,int n_r)
 984 {
 985 #define ZBS  DDZONECOMM_BUFSIZE
 986     rvec vbuf_s[DDZONECOMM_MAXZONE*ZBS];
 987     rvec vbuf_r[DDZONECOMM_MAXZONE*ZBS];
 988     int i;
 989
 990     for(i=0; i<n_s; i++)
 991     {
 992         vbuf_s[i*ZBS  ][0] = buf_s[i].min0;
 993         vbuf_s[i*ZBS  ][1] = buf_s[i].max1;
 994         vbuf_s[i*ZBS  ][2] = buf_s[i].min1;
 995         vbuf_s[i*ZBS+1][0] = buf_s[i].mch0;
 996         vbuf_s[i*ZBS+1][1] = buf_s[i].mch1;
 997         vbuf_s[i*ZBS+1][2] = 0;
 998         vbuf_s[i*ZBS+2][0] = buf_s[i].p1_0;
 999         vbuf_s[i*ZBS+2][1] = buf_s[i].p1_1;
1000         vbuf_s[i*ZBS+2][2] = 0;
1001     }
1002
1003     dd_sendrecv_rvec(dd, ddimind, direction,
1004                      vbuf_s, n_s*ZBS,
1005                      vbuf_r, n_r*ZBS);
1006
1007     for(i=0; i<n_r; i++)
1008     {
1009         buf_r[i].min0 = vbuf_r[i*ZBS  ][0];
1010         buf_r[i].max1 = vbuf_r[i*ZBS  ][1];
1011         buf_r[i].min1 = vbuf_r[i*ZBS  ][2];
1012         buf_r[i].mch0 = vbuf_r[i*ZBS+1][0];
1013         buf_r[i].mch1 = vbuf_r[i*ZBS+1][1];
1014         buf_r[i].p1_0 = vbuf_r[i*ZBS+2][0];
1015         buf_r[i].p1_1 = vbuf_r[i*ZBS+2][1];
1016     }
1017
1018 #undef ZBS
1019 }
1020
1021 static void dd_move_cellx(gmx_domdec_t *dd,gmx_ddbox_t *ddbox,
1022                           rvec cell_ns_x0,rvec cell_ns_x1)
1023 {
1024     int  d,d1,dim,dim1,pos,buf_size,i,j,k,p,npulse,npulse_min;
1025     gmx_ddzone_t *zp;
1026     gmx_ddzone_t buf_s[DDZONECOMM_MAXZONE];
1027     gmx_ddzone_t buf_r[DDZONECOMM_MAXZONE];
1028     gmx_ddzone_t buf_e[DDZONECOMM_MAXZONE];
1029     rvec extr_s[2],extr_r[2];
1030     rvec dh;
1031     real dist_d,c=0,det;
1032     gmx_domdec_comm_t *comm;
1033     gmx_bool bPBC,bUse;
1034
1035     comm = dd->comm;
1036
1037     for(d=1; d<dd->ndim; d++)
1038     {
1039         dim = dd->dim[d];
1040         zp = (d == 1) ? &comm->zone_d1[0] : &comm->zone_d2[0][0];
1041         zp->min0 = cell_ns_x0[dim];
1042         zp->max1 = cell_ns_x1[dim];
1043         zp->min1 = cell_ns_x1[dim];
1044         zp->mch0 = cell_ns_x0[dim];
1045         zp->mch1 = cell_ns_x1[dim];
1046         zp->p1_0 = cell_ns_x0[dim];
1047         zp->p1_1 = cell_ns_x1[dim];
1048     }
1049
1050     for(d=dd->ndim-2; d>=0; d--)
1051     {
1052         dim  = dd->dim[d];
1053         bPBC = (dim < ddbox->npbcdim);
1054
1055         /* Use an rvec to store two reals */
1056         extr_s[d][0] = comm->cell_f0[d+1];
1057         extr_s[d][1] = comm->cell_f1[d+1];
1058         extr_s[d][2] = comm->cell_f1[d+1];
1059
1060         pos = 0;
1061         /* Store the extremes in the backward sending buffer,
1062          * so the get updated separately from the forward communication.
1063          */
1064         for(d1=d; d1<dd->ndim-1; d1++)
1065         {
1066             /* We invert the order to be able to use the same loop for buf_e */
1067             buf_s[pos].min0 = extr_s[d1][1];
1068             buf_s[pos].max1 = extr_s[d1][0];
1069             buf_s[pos].min1 = extr_s[d1][2];
1070             buf_s[pos].mch0 = 0;
1071             buf_s[pos].mch1 = 0;
1072             /* Store the cell corner of the dimension we communicate along */
1073             buf_s[pos].p1_0 = comm->cell_x0[dim];
1074             buf_s[pos].p1_1 = 0;
1075             pos++;
1076         }
1077
1078         buf_s[pos] = (dd->ndim == 2) ? comm->zone_d1[0] : comm->zone_d2[0][0];
1079         pos++;
1080
1081         if (dd->ndim == 3 && d == 0)
1082         {
1083             buf_s[pos] = comm->zone_d2[0][1];
1084             pos++;
1085             buf_s[pos] = comm->zone_d1[0];
1086             pos++;
1087         }
1088
1089         /* We only need to communicate the extremes
1090          * in the forward direction
1091          */
1092         npulse = comm->cd[d].np;
1093         if (bPBC)
1094         {
1095             /* Take the minimum to avoid double communication */
1096             npulse_min = min(npulse,dd->nc[dim]-1-npulse);
1097         }
1098         else
1099         {
1100             /* Without PBC we should really not communicate over
1101              * the boundaries, but implementing that complicates
1102              * the communication setup and therefore we simply
1103              * do all communication, but ignore some data.
1104              */
1105             npulse_min = npulse;
1106         }
1107         for(p=0; p<npulse_min; p++)
1108         {
1109             /* Communicate the extremes forward */
1110             bUse = (bPBC || dd->ci[dim] > 0);
1111
1112             dd_sendrecv_rvec(dd, d, dddirForward,
1113                              extr_s+d, dd->ndim-d-1,
1114                              extr_r+d, dd->ndim-d-1);
1115
1116             if (bUse)
1117             {
1118                 for(d1=d; d1<dd->ndim-1; d1++)
1119                 {
1120                     extr_s[d1][0] = max(extr_s[d1][0],extr_r[d1][0]);
1121                     extr_s[d1][1] = min(extr_s[d1][1],extr_r[d1][1]);
1122                     extr_s[d1][2] = min(extr_s[d1][2],extr_r[d1][2]);
1123                 }
1124             }
1125         }
1126
1127         buf_size = pos;
1128         for(p=0; p<npulse; p++)
1129         {
1130             /* Communicate all the zone information backward */
1131             bUse = (bPBC || dd->ci[dim] < dd->nc[dim] - 1);
1132
1133             dd_sendrecv_ddzone(dd, d, dddirBackward,
1134                                buf_s, buf_size,
1135                                buf_r, buf_size);
1136
1137             clear_rvec(dh);
1138             if (p > 0)
1139             {
1140                 for(d1=d+1; d1<dd->ndim; d1++)
1141                 {
1142                     /* Determine the decrease of maximum required
1143                      * communication height along d1 due to the distance along d,
1144                      * this avoids a lot of useless atom communication.
1145                      */
1146                     dist_d = comm->cell_x1[dim] - buf_r[0].p1_0;
1147
1148                     if (ddbox->tric_dir[dim])
1149                     {
1150                         /* c is the off-diagonal coupling between the cell planes
1151                          * along directions d and d1.
1152                          */
1153                         c = ddbox->v[dim][dd->dim[d1]][dim];
1154                     }
1155                     else
1156                     {
1157                         c = 0;
1158                     }
1159                     det = (1 + c*c)*comm->cutoff*comm->cutoff - dist_d*dist_d;
1160                     if (det > 0)
1161                     {
1162                         dh[d1] = comm->cutoff - (c*dist_d + sqrt(det))/(1 + c*c);
1163                     }
1164                     else
1165                     {
1166                         /* A negative value signals out of range */
1167                         dh[d1] = -1;
1168                     }
1169                 }
1170             }
1171
1172             /* Accumulate the extremes over all pulses */
1173             for(i=0; i<buf_size; i++)
1174             {
1175                 if (p == 0)
1176                 {
1177                     buf_e[i] = buf_r[i];
1178                 }
1179                 else
1180                 {
1181                     if (bUse)
1182                     {
1183                         buf_e[i].min0 = min(buf_e[i].min0,buf_r[i].min0);
1184                         buf_e[i].max1 = max(buf_e[i].max1,buf_r[i].max1);
1185                         buf_e[i].min1 = min(buf_e[i].min1,buf_r[i].min1);
1186                     }
1187
1188                     if (dd->ndim == 3 && d == 0 && i == buf_size - 1)
1189                     {
1190                         d1 = 1;
1191                     }
1192                     else
1193                     {
1194                         d1 = d + 1;
1195                     }
1196                     if (bUse && dh[d1] >= 0)
1197                     {
1198                         buf_e[i].mch0 = max(buf_e[i].mch0,buf_r[i].mch0-dh[d1]);
1199                         buf_e[i].mch1 = max(buf_e[i].mch1,buf_r[i].mch1-dh[d1]);
1200                     }
1201                 }
1202                 /* Copy the received buffer to the send buffer,
1203                  * to pass the data through with the next pulse.
1204                  */
1205                 buf_s[i] = buf_r[i];
1206             }
1207             if (((bPBC || dd->ci[dim]+npulse < dd->nc[dim]) && p == npulse-1) ||
1208                 (!bPBC && dd->ci[dim]+1+p == dd->nc[dim]-1))
1209             {
1210                 /* Store the extremes */
1211                 pos = 0;
1212
1213                 for(d1=d; d1<dd->ndim-1; d1++)
1214                 {
1215                     extr_s[d1][1] = min(extr_s[d1][1],buf_e[pos].min0);
1216                     extr_s[d1][0] = max(extr_s[d1][0],buf_e[pos].max1);
1217                     extr_s[d1][2] = min(extr_s[d1][2],buf_e[pos].min1);
1218                     pos++;
1219                 }
1220
1221                 if (d == 1 || (d == 0 && dd->ndim == 3))
1222                 {
1223                     for(i=d; i<2; i++)
1224                     {
1225                         comm->zone_d2[1-d][i] = buf_e[pos];
1226                         pos++;
1227                     }
1228                 }
1229                 if (d == 0)
1230                 {
1231                     comm->zone_d1[1] = buf_e[pos];
1232                     pos++;
1233                 }
1234             }
1235         }
1236     }
1237
1238     if (dd->ndim >= 2)
1239     {
1240         dim = dd->dim[1];
1241         for(i=0; i<2; i++)
1242         {
1243             if (debug)
1244             {
1245                 print_ddzone(debug,1,i,0,&comm->zone_d1[i]);
1246             }
1247             cell_ns_x0[dim] = min(cell_ns_x0[dim],comm->zone_d1[i].min0);
1248             cell_ns_x1[dim] = max(cell_ns_x1[dim],comm->zone_d1[i].max1);
1249         }
1250     }
1251     if (dd->ndim >= 3)
1252     {
1253         dim = dd->dim[2];
1254         for(i=0; i<2; i++)
1255         {
1256             for(j=0; j<2; j++)
1257             {
1258                 if (debug)
1259                 {
1260                     print_ddzone(debug,2,i,j,&comm->zone_d2[i][j]);
1261                 }
1262                 cell_ns_x0[dim] = min(cell_ns_x0[dim],comm->zone_d2[i][j].min0);
1263                 cell_ns_x1[dim] = max(cell_ns_x1[dim],comm->zone_d2[i][j].max1);
1264             }
1265         }
1266     }
1267     for(d=1; d<dd->ndim; d++)
1268     {
1269         comm->cell_f_max0[d] = extr_s[d-1][0];
1270         comm->cell_f_min1[d] = extr_s[d-1][1];
1271         if (debug)
1272         {
1273             fprintf(debug,"Cell fraction d %d, max0 %f, min1 %f\n",
1274                     d,comm->cell_f_max0[d],comm->cell_f_min1[d]);
1275         }
1276     }
1277 }
1278
1279 static void dd_collect_cg(gmx_domdec_t *dd,
1280                           t_state *state_local)
1281 {
1282     gmx_domdec_master_t *ma=NULL;
1283     int buf2[2],*ibuf,i,ncg_home=0,*cg=NULL,nat_home=0;
1284     t_block *cgs_gl;
1285
1286     if (state_local->ddp_count == dd->comm->master_cg_ddp_count)
1287     {
1288         /* The master has the correct distribution */
1289         return;
1290     }
1291
1292     if (state_local->ddp_count == dd->ddp_count)
1293     {
1294         ncg_home = dd->ncg_home;
1295         cg       = dd->index_gl;
1296         nat_home = dd->nat_home;
1297     }
1298     else if (state_local->ddp_count_cg_gl == state_local->ddp_count)
1299     {
1300         cgs_gl = &dd->comm->cgs_gl;
1301
1302         ncg_home = state_local->ncg_gl;
1303         cg       = state_local->cg_gl;
1304         nat_home = 0;
1305         for(i=0; i<ncg_home; i++)
1306         {
1307             nat_home += cgs_gl->index[cg[i]+1] - cgs_gl->index[cg[i]];
1308         }
1309     }
1310     else
1311     {
1312         gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
1313     }
1314
1315     buf2[0] = dd->ncg_home;
1316     buf2[1] = dd->nat_home;
1317     if (DDMASTER(dd))
1318     {
1319         ma = dd->ma;
1320         ibuf = ma->ibuf;
1321     }
1322     else
1323     {
1324         ibuf = NULL;
1325     }
1326     /* Collect the charge group and atom counts on the master */
1327     dd_gather(dd,2*sizeof(int),buf2,ibuf);
1328
1329     if (DDMASTER(dd))
1330     {
1331         ma->index[0] = 0;
1332         for(i=0; i<dd->nnodes; i++)
1333         {
1334             ma->ncg[i] = ma->ibuf[2*i];
1335             ma->nat[i] = ma->ibuf[2*i+1];
1336             ma->index[i+1] = ma->index[i] + ma->ncg[i];
1337
1338         }
1339         /* Make byte counts and indices */
1340         for(i=0; i<dd->nnodes; i++)
1341         {
1342             ma->ibuf[i] = ma->ncg[i]*sizeof(int);
1343             ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
1344         }
1345         if (debug)
1346         {
1347             fprintf(debug,"Initial charge group distribution: ");
1348             for(i=0; i<dd->nnodes; i++)
1349                 fprintf(debug," %d",ma->ncg[i]);
1350             fprintf(debug,"\n");
1351         }
1352     }
1353
1354     /* Collect the charge group indices on the master */
1355     dd_gatherv(dd,
1356                dd->ncg_home*sizeof(int),dd->index_gl,
1357                DDMASTER(dd) ? ma->ibuf : NULL,
1358                DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
1359                DDMASTER(dd) ? ma->cg : NULL);
1360
1361     dd->comm->master_cg_ddp_count = state_local->ddp_count;
1362 }
1363
1364 static void dd_collect_vec_sendrecv(gmx_domdec_t *dd,
1365                                     rvec *lv,rvec *v)
1366 {
1367     gmx_domdec_master_t *ma;
1368     int  n,i,c,a,nalloc=0;
1369     rvec *buf=NULL;
1370     t_block *cgs_gl;
1371
1372     ma = dd->ma;
1373
1374     if (!DDMASTER(dd))
1375     {
1376 #ifdef GMX_MPI
1377         MPI_Send(lv,dd->nat_home*sizeof(rvec),MPI_BYTE,DDMASTERRANK(dd),
1378                  dd->rank,dd->mpi_comm_all);
1379 #endif
1380     } else {
1381         /* Copy the master coordinates to the global array */
1382         cgs_gl = &dd->comm->cgs_gl;
1383
1384         n = DDMASTERRANK(dd);
1385         a = 0;
1386         for(i=ma->index[n]; i<ma->index[n+1]; i++)
1387         {
1388             for(c=cgs_gl->index[ma->cg[i]]; c<cgs_gl->index[ma->cg[i]+1]; c++)
1389             {
1390                 copy_rvec(lv[a++],v[c]);
1391             }
1392         }
1393
1394         for(n=0; n<dd->nnodes; n++)
1395         {
1396             if (n != dd->rank)
1397             {
1398                 if (ma->nat[n] > nalloc)
1399                 {
1400                     nalloc = over_alloc_dd(ma->nat[n]);
1401                     srenew(buf,nalloc);
1402                 }
1403 #ifdef GMX_MPI
1404                 MPI_Recv(buf,ma->nat[n]*sizeof(rvec),MPI_BYTE,DDRANK(dd,n),
1405                          n,dd->mpi_comm_all,MPI_STATUS_IGNORE);
1406 #endif
1407                 a = 0;
1408                 for(i=ma->index[n]; i<ma->index[n+1]; i++)
1409                 {
1410                     for(c=cgs_gl->index[ma->cg[i]]; c<cgs_gl->index[ma->cg[i]+1]; c++)
1411                     {
1412                         copy_rvec(buf[a++],v[c]);
1413                     }
1414                 }
1415             }
1416         }
1417         sfree(buf);
1418     }
1419 }
1420
1421 static void get_commbuffer_counts(gmx_domdec_t *dd,
1422                                   int **counts,int **disps)
1423 {
1424     gmx_domdec_master_t *ma;
1425     int n;
1426
1427     ma = dd->ma;
1428
1429     /* Make the rvec count and displacment arrays */
1430     *counts  = ma->ibuf;
1431     *disps   = ma->ibuf + dd->nnodes;
1432     for(n=0; n<dd->nnodes; n++)
1433     {
1434         (*counts)[n] = ma->nat[n]*sizeof(rvec);
1435         (*disps)[n]  = (n == 0 ? 0 : (*disps)[n-1] + (*counts)[n-1]);
1436     }
1437 }
1438
1439 static void dd_collect_vec_gatherv(gmx_domdec_t *dd,
1440                                    rvec *lv,rvec *v)
1441 {
1442     gmx_domdec_master_t *ma;
1443     int  *rcounts=NULL,*disps=NULL;
1444     int  n,i,c,a;
1445     rvec *buf=NULL;
1446     t_block *cgs_gl;
1447
1448     ma = dd->ma;
1449
1450     if (DDMASTER(dd))
1451     {
1452         get_commbuffer_counts(dd,&rcounts,&disps);
1453
1454         buf = ma->vbuf;
1455     }
1456
1457     dd_gatherv(dd,dd->nat_home*sizeof(rvec),lv,rcounts,disps,buf);
1458
1459     if (DDMASTER(dd))
1460     {
1461         cgs_gl = &dd->comm->cgs_gl;
1462
1463         a = 0;
1464         for(n=0; n<dd->nnodes; n++)
1465         {
1466             for(i=ma->index[n]; i<ma->index[n+1]; i++)
1467             {
1468                 for(c=cgs_gl->index[ma->cg[i]]; c<cgs_gl->index[ma->cg[i]+1]; c++)
1469                 {
1470                     copy_rvec(buf[a++],v[c]);
1471                 }
1472             }
1473         }
1474     }
1475 }
1476
1477 void dd_collect_vec(gmx_domdec_t *dd,
1478                     t_state *state_local,rvec *lv,rvec *v)
1479 {
1480     gmx_domdec_master_t *ma;
1481     int  n,i,c,a,nalloc=0;
1482     rvec *buf=NULL;
1483
1484     dd_collect_cg(dd,state_local);
1485
1486     if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
1487     {
1488         dd_collect_vec_sendrecv(dd,lv,v);
1489     }
1490     else
1491     {
1492         dd_collect_vec_gatherv(dd,lv,v);
1493     }
1494 }
1495
1496
1497 void dd_collect_state(gmx_domdec_t *dd,
1498                       t_state *state_local,t_state *state)
1499 {
1500     int est,i,j,nh;
1501
1502     nh = state->nhchainlength;
1503
1504     if (DDMASTER(dd))
1505     {
1506         for (i=0;i<efptNR;i++) {
1507             state->lambda[i] = state_local->lambda[i];
1508         }
1509         state->fep_state = state_local->fep_state;
1510         state->veta = state_local->veta;
1511         state->vol0 = state_local->vol0;
1512         copy_mat(state_local->box,state->box);
1513         copy_mat(state_local->boxv,state->boxv);
1514         copy_mat(state_local->svir_prev,state->svir_prev);
1515         copy_mat(state_local->fvir_prev,state->fvir_prev);
1516         copy_mat(state_local->pres_prev,state->pres_prev);
1517
1518
1519         for(i=0; i<state_local->ngtc; i++)
1520         {
1521             for(j=0; j<nh; j++) {
1522                 state->nosehoover_xi[i*nh+j]        = state_local->nosehoover_xi[i*nh+j];
1523                 state->nosehoover_vxi[i*nh+j]       = state_local->nosehoover_vxi[i*nh+j];
1524             }
1525             state->therm_integral[i] = state_local->therm_integral[i];
1526         }
1527         for(i=0; i<state_local->nnhpres; i++)
1528         {
1529             for(j=0; j<nh; j++) {
1530                 state->nhpres_xi[i*nh+j]        = state_local->nhpres_xi[i*nh+j];
1531                 state->nhpres_vxi[i*nh+j]       = state_local->nhpres_vxi[i*nh+j];
1532             }
1533         }
1534     }
1535     for(est=0; est<estNR; est++)
1536     {
1537         if (EST_DISTR(est) && (state_local->flags & (1<<est)))
1538         {
1539             switch (est) {
1540             case estX:
1541                 dd_collect_vec(dd,state_local,state_local->x,state->x);
1542                 break;
1543             case estV:
1544                 dd_collect_vec(dd,state_local,state_local->v,state->v);
1545                 break;
1546             case estSDX:
1547                 dd_collect_vec(dd,state_local,state_local->sd_X,state->sd_X);
1548                 break;
1549             case estCGP:
1550                 dd_collect_vec(dd,state_local,state_local->cg_p,state->cg_p);
1551                 break;
1552             case estLD_RNG:
1553                 if (state->nrngi == 1)
1554                 {
1555                     if (DDMASTER(dd))
1556                     {
1557                         for(i=0; i<state_local->nrng; i++)
1558                         {
1559                             state->ld_rng[i] = state_local->ld_rng[i];
1560                         }
1561                     }
1562                 }
1563                 else
1564                 {
1565                     dd_gather(dd,state_local->nrng*sizeof(state->ld_rng[0]),
1566                               state_local->ld_rng,state->ld_rng);
1567                 }
1568                 break;
1569             case estLD_RNGI:
1570                 if (state->nrngi == 1)
1571                 {
1572                    if (DDMASTER(dd))
1573                     {
1574                         state->ld_rngi[0] = state_local->ld_rngi[0];
1575                     }
1576                 }
1577                 else
1578                 {
1579                     dd_gather(dd,sizeof(state->ld_rngi[0]),
1580                               state_local->ld_rngi,state->ld_rngi);
1581                 }
1582                 break;
1583             case estDISRE_INITF:
1584             case estDISRE_RM3TAV:
1585             case estORIRE_INITF:
1586             case estORIRE_DTAV:
1587                 break;
1588             default:
1589                 gmx_incons("Unknown state entry encountered in dd_collect_state");
1590             }
1591         }
1592     }
1593 }
1594
1595 static void dd_realloc_state(t_state *state,rvec **f,int nalloc)
1596 {
1597     int est;
1598
1599     if (debug)
1600     {
1601         fprintf(debug,"Reallocating state: currently %d, required %d, allocating %d\n",state->nalloc,nalloc,over_alloc_dd(nalloc));
1602     }
1603
1604     state->nalloc = over_alloc_dd(nalloc);
1605
1606     for(est=0; est<estNR; est++)
1607     {
1608         if (EST_DISTR(est) && (state->flags & (1<<est)))
1609         {
1610             switch(est) {
1611             case estX:
1612                 srenew(state->x,state->nalloc);
1613                 break;
1614             case estV:
1615                 srenew(state->v,state->nalloc);
1616                 break;
1617             case estSDX:
1618                 srenew(state->sd_X,state->nalloc);
1619                 break;
1620             case estCGP:
1621                 srenew(state->cg_p,state->nalloc);
1622                 break;
1623             case estLD_RNG:
1624             case estLD_RNGI:
1625             case estDISRE_INITF:
1626             case estDISRE_RM3TAV:
1627             case estORIRE_INITF:
1628             case estORIRE_DTAV:
1629                 /* No reallocation required */
1630                 break;
1631             default:
1632                 gmx_incons("Unknown state entry encountered in dd_realloc_state");
1633             }
1634         }
1635     }
1636
1637     if (f != NULL)
1638     {
1639         srenew(*f,state->nalloc);
1640     }
1641 }
1642
1643 static void dd_check_alloc_ncg(t_forcerec *fr,t_state *state,rvec **f,
1644                                int nalloc)
1645 {
1646     if (nalloc > fr->cg_nalloc)
1647     {
1648         if (debug)
1649         {
1650             fprintf(debug,"Reallocating forcerec: currently %d, required %d, allocating %d\n",fr->cg_nalloc,nalloc,over_alloc_dd(nalloc));
1651         }
1652         fr->cg_nalloc = over_alloc_dd(nalloc);
1653         srenew(fr->cginfo,fr->cg_nalloc);
1654         if (fr->cutoff_scheme == ecutsGROUP)
1655         {
1656             srenew(fr->cg_cm,fr->cg_nalloc);
1657         }
1658     }
1659     if (fr->cutoff_scheme == ecutsVERLET && nalloc > state->nalloc)
1660     {
1661         /* We don't use charge groups, we use x in state to set up
1662          * the atom communication.
1663          */
1664         dd_realloc_state(state,f,nalloc);
1665     }
1666 }
1667
1668 static void dd_distribute_vec_sendrecv(gmx_domdec_t *dd,t_block *cgs,
1669                                        rvec *v,rvec *lv)
1670 {
1671     gmx_domdec_master_t *ma;
1672     int  n,i,c,a,nalloc=0;
1673     rvec *buf=NULL;
1674
1675     if (DDMASTER(dd))
1676     {
1677         ma  = dd->ma;
1678
1679         for(n=0; n<dd->nnodes; n++)
1680         {
1681             if (n != dd->rank)
1682             {
1683                 if (ma->nat[n] > nalloc)
1684                 {
1685                     nalloc = over_alloc_dd(ma->nat[n]);
1686                     srenew(buf,nalloc);
1687                 }
1688                 /* Use lv as a temporary buffer */
1689                 a = 0;
1690                 for(i=ma->index[n]; i<ma->index[n+1]; i++)
1691                 {
1692                     for(c=cgs->index[ma->cg[i]]; c<cgs->index[ma->cg[i]+1]; c++)
1693                     {
1694                         copy_rvec(v[c],buf[a++]);
1695                     }
1696                 }
1697                 if (a != ma->nat[n])
1698                 {
1699                     gmx_fatal(FARGS,"Internal error a (%d) != nat (%d)",
1700                               a,ma->nat[n]);
1701                 }
1702
1703 #ifdef GMX_MPI
1704                 MPI_Send(buf,ma->nat[n]*sizeof(rvec),MPI_BYTE,
1705                          DDRANK(dd,n),n,dd->mpi_comm_all);
1706 #endif
1707             }
1708         }
1709         sfree(buf);
1710         n = DDMASTERRANK(dd);
1711         a = 0;
1712         for(i=ma->index[n]; i<ma->index[n+1]; i++)
1713         {
1714             for(c=cgs->index[ma->cg[i]]; c<cgs->index[ma->cg[i]+1]; c++)
1715             {
1716                 copy_rvec(v[c],lv[a++]);
1717             }
1718         }
1719     }
1720     else
1721     {
1722 #ifdef GMX_MPI
1723         MPI_Recv(lv,dd->nat_home*sizeof(rvec),MPI_BYTE,DDMASTERRANK(dd),
1724                  MPI_ANY_TAG,dd->mpi_comm_all,MPI_STATUS_IGNORE);
1725 #endif
1726     }
1727 }
1728
1729 static void dd_distribute_vec_scatterv(gmx_domdec_t *dd,t_block *cgs,
1730                                        rvec *v,rvec *lv)
1731 {
1732     gmx_domdec_master_t *ma;
1733     int  *scounts=NULL,*disps=NULL;
1734     int  n,i,c,a,nalloc=0;
1735     rvec *buf=NULL;
1736
1737     if (DDMASTER(dd))
1738     {
1739         ma  = dd->ma;
1740
1741         get_commbuffer_counts(dd,&scounts,&disps);
1742
1743         buf = ma->vbuf;
1744         a = 0;
1745         for(n=0; n<dd->nnodes; n++)
1746         {
1747             for(i=ma->index[n]; i<ma->index[n+1]; i++)
1748             {
1749                 for(c=cgs->index[ma->cg[i]]; c<cgs->index[ma->cg[i]+1]; c++)
1750                 {
1751                     copy_rvec(v[c],buf[a++]);
1752                 }
1753             }
1754         }
1755     }
1756
1757     dd_scatterv(dd,scounts,disps,buf,dd->nat_home*sizeof(rvec),lv);
1758 }
1759
1760 static void dd_distribute_vec(gmx_domdec_t *dd,t_block *cgs,rvec *v,rvec *lv)
1761 {
1762     if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
1763     {
1764         dd_distribute_vec_sendrecv(dd,cgs,v,lv);
1765     }
1766     else
1767     {
1768         dd_distribute_vec_scatterv(dd,cgs,v,lv);
1769     }
1770 }
1771
1772 static void dd_distribute_state(gmx_domdec_t *dd,t_block *cgs,
1773                                 t_state *state,t_state *state_local,
1774                                 rvec **f)
1775 {
1776     int  i,j,nh;
1777
1778     nh = state->nhchainlength;
1779
1780     if (DDMASTER(dd))
1781     {
1782         for(i=0;i<efptNR;i++)
1783         {
1784             state_local->lambda[i] = state->lambda[i];
1785         }
1786         state_local->fep_state = state->fep_state;
1787         state_local->veta   = state->veta;
1788         state_local->vol0   = state->vol0;
1789         copy_mat(state->box,state_local->box);
1790         copy_mat(state->box_rel,state_local->box_rel);
1791         copy_mat(state->boxv,state_local->boxv);
1792         copy_mat(state->svir_prev,state_local->svir_prev);
1793         copy_mat(state->fvir_prev,state_local->fvir_prev);
1794         for(i=0; i<state_local->ngtc; i++)
1795         {
1796             for(j=0; j<nh; j++) {
1797                 state_local->nosehoover_xi[i*nh+j]        = state->nosehoover_xi[i*nh+j];
1798                 state_local->nosehoover_vxi[i*nh+j]       = state->nosehoover_vxi[i*nh+j];
1799             }
1800             state_local->therm_integral[i] = state->therm_integral[i];
1801         }
1802         for(i=0; i<state_local->nnhpres; i++)
1803         {
1804             for(j=0; j<nh; j++) {
1805                 state_local->nhpres_xi[i*nh+j]        = state->nhpres_xi[i*nh+j];
1806                 state_local->nhpres_vxi[i*nh+j]       = state->nhpres_vxi[i*nh+j];
1807             }
1808         }
1809     }
1810     dd_bcast(dd,((efptNR)*sizeof(real)),state_local->lambda);
1811     dd_bcast(dd,sizeof(int),&state_local->fep_state);
1812     dd_bcast(dd,sizeof(real),&state_local->veta);
1813     dd_bcast(dd,sizeof(real),&state_local->vol0);
1814     dd_bcast(dd,sizeof(state_local->box),state_local->box);
1815     dd_bcast(dd,sizeof(state_local->box_rel),state_local->box_rel);
1816     dd_bcast(dd,sizeof(state_local->boxv),state_local->boxv);
1817     dd_bcast(dd,sizeof(state_local->svir_prev),state_local->svir_prev);
1818     dd_bcast(dd,sizeof(state_local->fvir_prev),state_local->fvir_prev);
1819     dd_bcast(dd,((state_local->ngtc*nh)*sizeof(double)),state_local->nosehoover_xi);
1820     dd_bcast(dd,((state_local->ngtc*nh)*sizeof(double)),state_local->nosehoover_vxi);
1821     dd_bcast(dd,state_local->ngtc*sizeof(double),state_local->therm_integral);
1822     dd_bcast(dd,((state_local->nnhpres*nh)*sizeof(double)),state_local->nhpres_xi);
1823     dd_bcast(dd,((state_local->nnhpres*nh)*sizeof(double)),state_local->nhpres_vxi);
1824
1825     if (dd->nat_home > state_local->nalloc)
1826     {
1827         dd_realloc_state(state_local,f,dd->nat_home);
1828     }
1829     for(i=0; i<estNR; i++)
1830     {
1831         if (EST_DISTR(i) && (state_local->flags & (1<<i)))
1832         {
1833             switch (i) {
1834             case estX:
1835                 dd_distribute_vec(dd,cgs,state->x,state_local->x);
1836                 break;
1837             case estV:
1838                 dd_distribute_vec(dd,cgs,state->v,state_local->v);
1839                 break;
1840             case estSDX:
1841                 dd_distribute_vec(dd,cgs,state->sd_X,state_local->sd_X);
1842                 break;
1843             case estCGP:
1844                 dd_distribute_vec(dd,cgs,state->cg_p,state_local->cg_p);
1845                 break;
1846             case estLD_RNG:
1847                 if (state->nrngi == 1)
1848                 {
1849                     dd_bcastc(dd,
1850                               state_local->nrng*sizeof(state_local->ld_rng[0]),
1851                               state->ld_rng,state_local->ld_rng);
1852                 }
1853                 else
1854                 {
1855                     dd_scatter(dd,
1856                                state_local->nrng*sizeof(state_local->ld_rng[0]),
1857                                state->ld_rng,state_local->ld_rng);
1858                 }
1859                 break;
1860             case estLD_RNGI:
1861                 if (state->nrngi == 1)
1862                 {
1863                     dd_bcastc(dd,sizeof(state_local->ld_rngi[0]),
1864                               state->ld_rngi,state_local->ld_rngi);
1865                 }
1866                 else
1867                 {
1868                      dd_scatter(dd,sizeof(state_local->ld_rngi[0]),
1869                                state->ld_rngi,state_local->ld_rngi);
1870                 }
1871                 break;
1872             case estDISRE_INITF:
1873             case estDISRE_RM3TAV:
1874             case estORIRE_INITF:
1875             case estORIRE_DTAV:
1876                 /* Not implemented yet */
1877                 break;
1878             default:
1879                 gmx_incons("Unknown state entry encountered in dd_distribute_state");
1880             }
1881         }
1882     }
1883 }
1884
1885 static char dim2char(int dim)
1886 {
1887     char c='?';
1888
1889     switch (dim)
1890     {
1891     case XX: c = 'X'; break;
1892     case YY: c = 'Y'; break;
1893     case ZZ: c = 'Z'; break;
1894     default: gmx_fatal(FARGS,"Unknown dim %d",dim);
1895     }
1896
1897     return c;
1898 }
1899
1900 static void write_dd_grid_pdb(const char *fn,gmx_large_int_t step,
1901                               gmx_domdec_t *dd,matrix box,gmx_ddbox_t *ddbox)
1902 {
1903     rvec grid_s[2],*grid_r=NULL,cx,r;
1904     char fname[STRLEN],format[STRLEN],buf[22];
1905     FILE *out;
1906     int  a,i,d,z,y,x;
1907     matrix tric;
1908     real vol;
1909
1910     copy_rvec(dd->comm->cell_x0,grid_s[0]);
1911     copy_rvec(dd->comm->cell_x1,grid_s[1]);
1912
1913     if (DDMASTER(dd))
1914     {
1915         snew(grid_r,2*dd->nnodes);
1916     }
1917
1918     dd_gather(dd,2*sizeof(rvec),grid_s[0],DDMASTER(dd) ? grid_r[0] : NULL);
1919
1920     if (DDMASTER(dd))
1921     {
1922         for(d=0; d<DIM; d++)
1923         {
1924             for(i=0; i<DIM; i++)
1925             {
1926                 if (d == i)
1927                 {
1928                     tric[d][i] = 1;
1929                 }
1930                 else
1931                 {
1932                     if (d < ddbox->npbcdim && dd->nc[d] > 1)
1933                     {
1934                         tric[d][i] = box[i][d]/box[i][i];
1935                     }
1936                     else
1937                     {
1938                         tric[d][i] = 0;
1939                     }
1940                 }
1941             }
1942         }
1943         sprintf(fname,"%s_%s.pdb",fn,gmx_step_str(step,buf));
1944         sprintf(format,"%s%s\n",pdbformat,"%6.2f%6.2f");
1945         out = gmx_fio_fopen(fname,"w");
1946         gmx_write_pdb_box(out,dd->bScrewPBC ? epbcSCREW : epbcXYZ,box);
1947         a = 1;
1948         for(i=0; i<dd->nnodes; i++)
1949         {
1950             vol = dd->nnodes/(box[XX][XX]*box[YY][YY]*box[ZZ][ZZ]);
1951             for(d=0; d<DIM; d++)
1952             {
1953                 vol *= grid_r[i*2+1][d] - grid_r[i*2][d];
1954             }
1955             for(z=0; z<2; z++)
1956             {
1957                 for(y=0; y<2; y++)
1958                 {
1959                     for(x=0; x<2; x++)
1960                     {
1961                         cx[XX] = grid_r[i*2+x][XX];
1962                         cx[YY] = grid_r[i*2+y][YY];
1963                         cx[ZZ] = grid_r[i*2+z][ZZ];
1964                         mvmul(tric,cx,r);
1965                         fprintf(out,format,"ATOM",a++,"CA","GLY",' ',1+i,
1966                                 10*r[XX],10*r[YY],10*r[ZZ],1.0,vol);
1967                     }
1968                 }
1969             }
1970             for(d=0; d<DIM; d++)
1971             {
1972                 for(x=0; x<4; x++)
1973                 {
1974                     switch(d)
1975                     {
1976                     case 0: y = 1 + i*8 + 2*x; break;
1977                     case 1: y = 1 + i*8 + 2*x - (x % 2); break;
1978                     case 2: y = 1 + i*8 + x; break;
1979                     }
1980                     fprintf(out,"%6s%5d%5d\n","CONECT",y,y+(1<<d));
1981                 }
1982             }
1983         }
1984         gmx_fio_fclose(out);
1985         sfree(grid_r);
1986     }
1987 }
1988
1989 void write_dd_pdb(const char *fn,gmx_large_int_t step,const char *title,
1990                   gmx_mtop_t *mtop,t_commrec *cr,
1991                   int natoms,rvec x[],matrix box)
1992 {
1993     char fname[STRLEN],format[STRLEN],format4[STRLEN],buf[22];
1994     FILE *out;
1995     int  i,ii,resnr,c;
1996     char *atomname,*resname;
1997     real b;
1998     gmx_domdec_t *dd;
1999
2000     dd = cr->dd;
2001     if (natoms == -1)
2002     {
2003         natoms = dd->comm->nat[ddnatVSITE];
2004     }
2005
2006     sprintf(fname,"%s_%s_n%d.pdb",fn,gmx_step_str(step,buf),cr->sim_nodeid);
2007
2008     sprintf(format,"%s%s\n",pdbformat,"%6.2f%6.2f");
2009     sprintf(format4,"%s%s\n",pdbformat4,"%6.2f%6.2f");
2010
2011     out = gmx_fio_fopen(fname,"w");
2012
2013     fprintf(out,"TITLE     %s\n",title);
2014     gmx_write_pdb_box(out,dd->bScrewPBC ? epbcSCREW : epbcXYZ,box);
2015     for(i=0; i<natoms; i++)
2016     {
2017         ii = dd->gatindex[i];
2018         gmx_mtop_atominfo_global(mtop,ii,&atomname,&resnr,&resname);
2019         if (i < dd->comm->nat[ddnatZONE])
2020         {
2021             c = 0;
2022             while (i >= dd->cgindex[dd->comm->zones.cg_range[c+1]])
2023             {
2024                 c++;
2025             }
2026             b = c;
2027         }
2028         else if (i < dd->comm->nat[ddnatVSITE])
2029         {
2030             b = dd->comm->zones.n;
2031         }
2032         else
2033         {
2034             b = dd->comm->zones.n + 1;
2035         }
2036         fprintf(out,strlen(atomname)<4 ? format : format4,
2037                 "ATOM",(ii+1)%100000,
2038                 atomname,resname,' ',resnr%10000,' ',
2039                 10*x[i][XX],10*x[i][YY],10*x[i][ZZ],1.0,b);
2040     }
2041     fprintf(out,"TER\n");
2042
2043     gmx_fio_fclose(out);
2044 }
2045
2046 real dd_cutoff_mbody(gmx_domdec_t *dd)
2047 {
2048     gmx_domdec_comm_t *comm;
2049     int  di;
2050     real r;
2051
2052     comm = dd->comm;
2053
2054     r = -1;
2055     if (comm->bInterCGBondeds)
2056     {
2057         if (comm->cutoff_mbody > 0)
2058         {
2059             r = comm->cutoff_mbody;
2060         }
2061         else
2062         {
2063             /* cutoff_mbody=0 means we do not have DLB */
2064             r = comm->cellsize_min[dd->dim[0]];
2065             for(di=1; di<dd->ndim; di++)
2066             {
2067                 r = min(r,comm->cellsize_min[dd->dim[di]]);
2068             }
2069             if (comm->bBondComm)
2070             {
2071                 r = max(r,comm->cutoff_mbody);
2072             }
2073             else
2074             {
2075                 r = min(r,comm->cutoff);
2076             }
2077         }
2078     }
2079
2080     return r;
2081 }
2082
2083 real dd_cutoff_twobody(gmx_domdec_t *dd)
2084 {
2085     real r_mb;
2086
2087     r_mb = dd_cutoff_mbody(dd);
2088
2089     return max(dd->comm->cutoff,r_mb);
2090 }
2091
2092
2093 static void dd_cart_coord2pmecoord(gmx_domdec_t *dd,ivec coord,ivec coord_pme)
2094 {
2095     int nc,ntot;
2096
2097     nc   = dd->nc[dd->comm->cartpmedim];
2098     ntot = dd->comm->ntot[dd->comm->cartpmedim];
2099     copy_ivec(coord,coord_pme);
2100     coord_pme[dd->comm->cartpmedim] =
2101         nc + (coord[dd->comm->cartpmedim]*(ntot - nc) + (ntot - nc)/2)/nc;
2102 }
2103
2104 static int low_ddindex2pmeindex(int ndd,int npme,int ddindex)
2105 {
2106     /* Here we assign a PME node to communicate with this DD node
2107      * by assuming that the major index of both is x.
2108      * We add cr->npmenodes/2 to obtain an even distribution.
2109      */
2110     return (ddindex*npme + npme/2)/ndd;
2111 }
2112
2113 static int ddindex2pmeindex(const gmx_domdec_t *dd,int ddindex)
2114 {
2115     return low_ddindex2pmeindex(dd->nnodes,dd->comm->npmenodes,ddindex);
2116 }
2117
2118 static int cr_ddindex2pmeindex(const t_commrec *cr,int ddindex)
2119 {
2120     return low_ddindex2pmeindex(cr->dd->nnodes,cr->npmenodes,ddindex);
2121 }
2122
2123 static int *dd_pmenodes(t_commrec *cr)
2124 {
2125     int *pmenodes;
2126     int n,i,p0,p1;
2127
2128     snew(pmenodes,cr->npmenodes);
2129     n = 0;
2130     for(i=0; i<cr->dd->nnodes; i++) {
2131         p0 = cr_ddindex2pmeindex(cr,i);
2132         p1 = cr_ddindex2pmeindex(cr,i+1);
2133         if (i+1 == cr->dd->nnodes || p1 > p0) {
2134             if (debug)
2135                 fprintf(debug,"pmenode[%d] = %d\n",n,i+1+n);
2136             pmenodes[n] = i + 1 + n;
2137             n++;
2138         }
2139     }
2140
2141     return pmenodes;
2142 }
2143
2144 static int gmx_ddcoord2pmeindex(t_commrec *cr,int x,int y,int z)
2145 {
2146     gmx_domdec_t *dd;
2147     ivec coords,coords_pme,nc;
2148     int  slab;
2149
2150     dd = cr->dd;
2151     /*
2152       if (dd->comm->bCartesian) {
2153       gmx_ddindex2xyz(dd->nc,ddindex,coords);
2154       dd_coords2pmecoords(dd,coords,coords_pme);
2155       copy_ivec(dd->ntot,nc);
2156       nc[dd->cartpmedim]         -= dd->nc[dd->cartpmedim];
2157       coords_pme[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
2158
2159       slab = (coords_pme[XX]*nc[YY] + coords_pme[YY])*nc[ZZ] + coords_pme[ZZ];
2160       } else {
2161       slab = (ddindex*cr->npmenodes + cr->npmenodes/2)/dd->nnodes;
2162       }
2163     */
2164     coords[XX] = x;
2165     coords[YY] = y;
2166     coords[ZZ] = z;
2167     slab = ddindex2pmeindex(dd,dd_index(dd->nc,coords));
2168
2169     return slab;
2170 }
2171
2172 static int ddcoord2simnodeid(t_commrec *cr,int x,int y,int z)
2173 {
2174     gmx_domdec_comm_t *comm;
2175     ivec coords;
2176     int  ddindex,nodeid=-1;
2177
2178     comm = cr->dd->comm;
2179
2180     coords[XX] = x;
2181     coords[YY] = y;
2182     coords[ZZ] = z;
2183     if (comm->bCartesianPP_PME)
2184     {
2185 #ifdef GMX_MPI
2186         MPI_Cart_rank(cr->mpi_comm_mysim,coords,&nodeid);
2187 #endif
2188     }
2189     else
2190     {
2191         ddindex = dd_index(cr->dd->nc,coords);
2192         if (comm->bCartesianPP)
2193         {
2194             nodeid = comm->ddindex2simnodeid[ddindex];
2195         }
2196         else
2197         {
2198             if (comm->pmenodes)
2199             {
2200                 nodeid = ddindex + gmx_ddcoord2pmeindex(cr,x,y,z);
2201             }
2202             else
2203             {
2204                 nodeid = ddindex;
2205             }
2206         }
2207     }
2208
2209     return nodeid;
2210 }
2211
2212 static int dd_simnode2pmenode(t_commrec *cr,int sim_nodeid)
2213 {
2214     gmx_domdec_t *dd;
2215     gmx_domdec_comm_t *comm;
2216     ivec coord,coord_pme;
2217     int  i;
2218     int  pmenode=-1;
2219
2220     dd = cr->dd;
2221     comm = dd->comm;
2222
2223     /* This assumes a uniform x domain decomposition grid cell size */
2224     if (comm->bCartesianPP_PME)
2225     {
2226 #ifdef GMX_MPI
2227         MPI_Cart_coords(cr->mpi_comm_mysim,sim_nodeid,DIM,coord);
2228         if (coord[comm->cartpmedim] < dd->nc[comm->cartpmedim])
2229         {
2230             /* This is a PP node */
2231             dd_cart_coord2pmecoord(dd,coord,coord_pme);
2232             MPI_Cart_rank(cr->mpi_comm_mysim,coord_pme,&pmenode);
2233         }
2234 #endif
2235     }
2236     else if (comm->bCartesianPP)
2237     {
2238         if (sim_nodeid < dd->nnodes)
2239         {
2240             pmenode = dd->nnodes + ddindex2pmeindex(dd,sim_nodeid);
2241         }
2242     }
2243     else
2244     {
2245         /* This assumes DD cells with identical x coordinates
2246          * are numbered sequentially.
2247          */
2248         if (dd->comm->pmenodes == NULL)
2249         {
2250             if (sim_nodeid < dd->nnodes)
2251             {
2252                 /* The DD index equals the nodeid */
2253                 pmenode = dd->nnodes + ddindex2pmeindex(dd,sim_nodeid);
2254             }
2255         }
2256         else
2257         {
2258             i = 0;
2259             while (sim_nodeid > dd->comm->pmenodes[i])
2260             {
2261                 i++;
2262             }
2263             if (sim_nodeid < dd->comm->pmenodes[i])
2264             {
2265                 pmenode = dd->comm->pmenodes[i];
2266             }
2267         }
2268     }
2269
2270     return pmenode;
2271 }
2272
2273 gmx_bool gmx_pmeonlynode(t_commrec *cr,int sim_nodeid)
2274 {
2275     gmx_bool bPMEOnlyNode;
2276
2277     if (DOMAINDECOMP(cr))
2278     {
2279         bPMEOnlyNode = (dd_simnode2pmenode(cr,sim_nodeid) == -1);
2280     }
2281     else
2282     {
2283         bPMEOnlyNode = FALSE;
2284     }
2285
2286     return bPMEOnlyNode;
2287 }
2288
2289 void get_pme_ddnodes(t_commrec *cr,int pmenodeid,
2290                      int *nmy_ddnodes,int **my_ddnodes,int *node_peer)
2291 {
2292     gmx_domdec_t *dd;
2293     int x,y,z;
2294     ivec coord,coord_pme;
2295
2296     dd = cr->dd;
2297
2298     snew(*my_ddnodes,(dd->nnodes+cr->npmenodes-1)/cr->npmenodes);
2299
2300     *nmy_ddnodes = 0;
2301     for(x=0; x<dd->nc[XX]; x++)
2302     {
2303         for(y=0; y<dd->nc[YY]; y++)
2304         {
2305             for(z=0; z<dd->nc[ZZ]; z++)
2306             {
2307                 if (dd->comm->bCartesianPP_PME)
2308                 {
2309                     coord[XX] = x;
2310                     coord[YY] = y;
2311                     coord[ZZ] = z;
2312                     dd_cart_coord2pmecoord(dd,coord,coord_pme);
2313                     if (dd->ci[XX] == coord_pme[XX] &&
2314                         dd->ci[YY] == coord_pme[YY] &&
2315                         dd->ci[ZZ] == coord_pme[ZZ])
2316                         (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr,x,y,z);
2317                 }
2318                 else
2319                 {
2320                     /* The slab corresponds to the nodeid in the PME group */
2321                     if (gmx_ddcoord2pmeindex(cr,x,y,z) == pmenodeid)
2322                     {
2323                         (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr,x,y,z);
2324                     }
2325                 }
2326             }
2327         }
2328     }
2329
2330     /* The last PP-only node is the peer node */
2331     *node_peer = (*my_ddnodes)[*nmy_ddnodes-1];
2332
2333     if (debug)
2334     {
2335         fprintf(debug,"Receive coordinates from PP nodes:");
2336         for(x=0; x<*nmy_ddnodes; x++)
2337         {
2338             fprintf(debug," %d",(*my_ddnodes)[x]);
2339         }
2340         fprintf(debug,"\n");
2341     }
2342 }
2343
2344 static gmx_bool receive_vir_ener(t_commrec *cr)
2345 {
2346     gmx_domdec_comm_t *comm;
2347     int  pmenode,coords[DIM],rank;
2348     gmx_bool bReceive;
2349
2350     bReceive = TRUE;
2351     if (cr->npmenodes < cr->dd->nnodes)
2352     {
2353         comm = cr->dd->comm;
2354         if (comm->bCartesianPP_PME)
2355         {
2356             pmenode = dd_simnode2pmenode(cr,cr->sim_nodeid);
2357 #ifdef GMX_MPI
2358             MPI_Cart_coords(cr->mpi_comm_mysim,cr->sim_nodeid,DIM,coords);
2359             coords[comm->cartpmedim]++;
2360             if (coords[comm->cartpmedim] < cr->dd->nc[comm->cartpmedim])
2361             {
2362                 MPI_Cart_rank(cr->mpi_comm_mysim,coords,&rank);
2363                 if (dd_simnode2pmenode(cr,rank) == pmenode)
2364                 {
2365                     /* This is not the last PP node for pmenode */
2366                     bReceive = FALSE;
2367                 }
2368             }
2369 #endif
2370         }
2371         else
2372         {
2373             pmenode = dd_simnode2pmenode(cr,cr->sim_nodeid);
2374             if (cr->sim_nodeid+1 < cr->nnodes &&
2375                 dd_simnode2pmenode(cr,cr->sim_nodeid+1) == pmenode)
2376             {
2377                 /* This is not the last PP node for pmenode */
2378                 bReceive = FALSE;
2379             }
2380         }
2381     }
2382
2383     return bReceive;
2384 }
2385
2386 static void set_zones_ncg_home(gmx_domdec_t *dd)
2387 {
2388     gmx_domdec_zones_t *zones;
2389     int i;
2390
2391     zones = &dd->comm->zones;
2392
2393     zones->cg_range[0] = 0;
2394     for(i=1; i<zones->n+1; i++)
2395     {
2396         zones->cg_range[i] = dd->ncg_home;
2397     }
2398 }
2399
2400 static void rebuild_cgindex(gmx_domdec_t *dd,
2401                             const int *gcgs_index,t_state *state)
2402 {
2403     int nat,i,*ind,*dd_cg_gl,*cgindex,cg_gl;
2404
2405     ind = state->cg_gl;
2406     dd_cg_gl = dd->index_gl;
2407     cgindex  = dd->cgindex;
2408     nat = 0;
2409     cgindex[0] = nat;
2410     for(i=0; i<state->ncg_gl; i++)
2411     {
2412         cgindex[i] = nat;
2413         cg_gl = ind[i];
2414         dd_cg_gl[i] = cg_gl;
2415         nat += gcgs_index[cg_gl+1] - gcgs_index[cg_gl];
2416     }
2417     cgindex[i] = nat;
2418
2419     dd->ncg_home = state->ncg_gl;
2420     dd->nat_home = nat;
2421
2422     set_zones_ncg_home(dd);
2423 }
2424
2425 static int ddcginfo(const cginfo_mb_t *cginfo_mb,int cg)
2426 {
2427     while (cg >= cginfo_mb->cg_end)
2428     {
2429         cginfo_mb++;
2430     }
2431
2432     return cginfo_mb->cginfo[(cg - cginfo_mb->cg_start) % cginfo_mb->cg_mod];
2433 }
2434
2435 static void dd_set_cginfo(int *index_gl,int cg0,int cg1,
2436                           t_forcerec *fr,char *bLocalCG)
2437 {
2438     cginfo_mb_t *cginfo_mb;
2439     int *cginfo;
2440     int cg;
2441
2442     if (fr != NULL)
2443     {
2444         cginfo_mb = fr->cginfo_mb;
2445         cginfo    = fr->cginfo;
2446
2447         for(cg=cg0; cg<cg1; cg++)
2448         {
2449             cginfo[cg] = ddcginfo(cginfo_mb,index_gl[cg]);
2450         }
2451     }
2452
2453     if (bLocalCG != NULL)
2454     {
2455         for(cg=cg0; cg<cg1; cg++)
2456         {
2457             bLocalCG[index_gl[cg]] = TRUE;
2458         }
2459     }
2460 }
2461
2462 static void make_dd_indices(gmx_domdec_t *dd,
2463                             const int *gcgs_index,int cg_start)
2464 {
2465     int nzone,zone,zone1,cg0,cg1,cg1_p1,cg,cg_gl,a,a_gl;
2466     int *zone2cg,*zone_ncg1,*index_gl,*gatindex;
2467     gmx_ga2la_t *ga2la;
2468     char *bLocalCG;
2469     gmx_bool bCGs;
2470
2471     bLocalCG = dd->comm->bLocalCG;
2472
2473     if (dd->nat_tot > dd->gatindex_nalloc)
2474     {
2475         dd->gatindex_nalloc = over_alloc_dd(dd->nat_tot);
2476         srenew(dd->gatindex,dd->gatindex_nalloc);
2477     }
2478
2479     nzone      = dd->comm->zones.n;
2480     zone2cg    = dd->comm->zones.cg_range;
2481     zone_ncg1  = dd->comm->zone_ncg1;
2482     index_gl   = dd->index_gl;
2483     gatindex   = dd->gatindex;
2484     bCGs       = dd->comm->bCGs;
2485
2486     if (zone2cg[1] != dd->ncg_home)
2487     {
2488         gmx_incons("dd->ncg_zone is not up to date");
2489     }
2490
2491     /* Make the local to global and global to local atom index */
2492     a = dd->cgindex[cg_start];
2493     for(zone=0; zone<nzone; zone++)
2494     {
2495         if (zone == 0)
2496         {
2497             cg0 = cg_start;
2498         }
2499         else
2500         {
2501             cg0 = zone2cg[zone];
2502         }
2503         cg1    = zone2cg[zone+1];
2504         cg1_p1 = cg0 + zone_ncg1[zone];
2505
2506         for(cg=cg0; cg<cg1; cg++)
2507         {
2508             zone1 = zone;
2509             if (cg >= cg1_p1)
2510             {
2511                 /* Signal that this cg is from more than one pulse away */
2512                 zone1 += nzone;
2513             }
2514             cg_gl = index_gl[cg];
2515             if (bCGs)
2516             {
2517                 for(a_gl=gcgs_index[cg_gl]; a_gl<gcgs_index[cg_gl+1]; a_gl++)
2518                 {
2519                     gatindex[a] = a_gl;
2520                     ga2la_set(dd->ga2la,a_gl,a,zone1);
2521                     a++;
2522                 }
2523             }
2524             else
2525             {
2526                 gatindex[a] = cg_gl;
2527                 ga2la_set(dd->ga2la,cg_gl,a,zone1);
2528                 a++;
2529             }
2530         }
2531     }
2532 }
2533
2534 static int check_bLocalCG(gmx_domdec_t *dd,int ncg_sys,const char *bLocalCG,
2535                           const char *where)
2536 {
2537     int ncg,i,ngl,nerr;
2538
2539     nerr = 0;
2540     if (bLocalCG == NULL)
2541     {
2542         return nerr;
2543     }
2544     for(i=0; i<dd->ncg_tot; i++)
2545     {
2546         if (!bLocalCG[dd->index_gl[i]])
2547         {
2548             fprintf(stderr,
2549                     "DD node %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n",dd->rank,where,i+1,dd->index_gl[i]+1,dd->ncg_home);
2550             nerr++;
2551         }
2552     }
2553     ngl = 0;
2554     for(i=0; i<ncg_sys; i++)
2555     {
2556         if (bLocalCG[i])
2557         {
2558             ngl++;
2559         }
2560     }
2561     if (ngl != dd->ncg_tot)
2562     {
2563         fprintf(stderr,"DD node %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n",dd->rank,where,ngl,dd->ncg_tot);
2564         nerr++;
2565     }
2566
2567     return nerr;
2568 }
2569
2570 static void check_index_consistency(gmx_domdec_t *dd,
2571                                     int natoms_sys,int ncg_sys,
2572                                     const char *where)
2573 {
2574     int  nerr,ngl,i,a,cell;
2575     int  *have;
2576
2577     nerr = 0;
2578
2579     if (dd->comm->DD_debug > 1)
2580     {
2581         snew(have,natoms_sys);
2582         for(a=0; a<dd->nat_tot; a++)
2583         {
2584             if (have[dd->gatindex[a]] > 0)
2585             {
2586                 fprintf(stderr,"DD node %d: global atom %d occurs twice: index %d and %d\n",dd->rank,dd->gatindex[a]+1,have[dd->gatindex[a]],a+1);
2587             }
2588             else
2589             {
2590                 have[dd->gatindex[a]] = a + 1;
2591             }
2592         }
2593         sfree(have);
2594     }
2595
2596     snew(have,dd->nat_tot);
2597
2598     ngl  = 0;
2599     for(i=0; i<natoms_sys; i++)
2600     {
2601         if (ga2la_get(dd->ga2la,i,&a,&cell))
2602         {
2603             if (a >= dd->nat_tot)
2604             {
2605                 fprintf(stderr,"DD node %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n",dd->rank,i+1,a+1,dd->nat_tot);
2606                 nerr++;
2607             }
2608             else
2609             {
2610                 have[a] = 1;
2611                 if (dd->gatindex[a] != i)
2612                 {
2613                     fprintf(stderr,"DD node %d: global atom %d marked as local atom %d, which has global atom index %d\n",dd->rank,i+1,a+1,dd->gatindex[a]+1);
2614                     nerr++;
2615                 }
2616             }
2617             ngl++;
2618         }
2619     }
2620     if (ngl != dd->nat_tot)
2621     {
2622         fprintf(stderr,
2623                 "DD node %d, %s: %d global atom indices, %d local atoms\n",
2624                 dd->rank,where,ngl,dd->nat_tot);
2625     }
2626     for(a=0; a<dd->nat_tot; a++)
2627     {
2628         if (have[a] == 0)
2629         {
2630             fprintf(stderr,
2631                     "DD node %d, %s: local atom %d, global %d has no global index\n",
2632                     dd->rank,where,a+1,dd->gatindex[a]+1);
2633         }
2634     }
2635     sfree(have);
2636
2637     nerr += check_bLocalCG(dd,ncg_sys,dd->comm->bLocalCG,where);
2638
2639     if (nerr > 0) {
2640         gmx_fatal(FARGS,"DD node %d, %s: %d atom/cg index inconsistencies",
2641                   dd->rank,where,nerr);
2642     }
2643 }
2644
2645 static void clear_dd_indices(gmx_domdec_t *dd,int cg_start,int a_start)
2646 {
2647     int  i;
2648     char *bLocalCG;
2649
2650     if (a_start == 0)
2651     {
2652         /* Clear the whole list without searching */
2653         ga2la_clear(dd->ga2la);
2654     }
2655     else
2656     {
2657         for(i=a_start; i<dd->nat_tot; i++)
2658         {
2659             ga2la_del(dd->ga2la,dd->gatindex[i]);
2660         }
2661     }
2662
2663     bLocalCG = dd->comm->bLocalCG;
2664     if (bLocalCG)
2665     {
2666         for(i=cg_start; i<dd->ncg_tot; i++)
2667         {
2668             bLocalCG[dd->index_gl[i]] = FALSE;
2669         }
2670     }
2671
2672     dd_clear_local_vsite_indices(dd);
2673
2674     if (dd->constraints)
2675     {
2676         dd_clear_local_constraint_indices(dd);
2677     }
2678 }
2679
2680 static real grid_jump_limit(gmx_domdec_comm_t *comm,real cutoff,
2681                             int dim_ind)
2682 {
2683     real grid_jump_limit;
2684
2685     /* The distance between the boundaries of cells at distance
2686      * x+-1,y+-1 or y+-1,z+-1 is limited by the cut-off restrictions
2687      * and by the fact that cells should not be shifted by more than
2688      * half their size, such that cg's only shift by one cell
2689      * at redecomposition.
2690      */
2691     grid_jump_limit = comm->cellsize_limit;
2692     if (!comm->bVacDLBNoLimit)
2693     {
2694         grid_jump_limit = max(grid_jump_limit,
2695                               cutoff/comm->cd[dim_ind].np);
2696     }
2697
2698     return grid_jump_limit;
2699 }
2700
2701 static gmx_bool check_grid_jump(gmx_large_int_t step,
2702                                 gmx_domdec_t *dd,
2703                                 real cutoff,
2704                                 gmx_ddbox_t *ddbox,
2705                                 gmx_bool bFatal)
2706 {
2707     gmx_domdec_comm_t *comm;
2708     int  d,dim;
2709     real limit,bfac;
2710     gmx_bool bInvalid;
2711
2712     bInvalid = FALSE;
2713
2714     comm = dd->comm;
2715
2716     for(d=1; d<dd->ndim; d++)
2717     {
2718         dim = dd->dim[d];
2719         limit = grid_jump_limit(comm,cutoff,d);
2720         bfac = ddbox->box_size[dim];
2721         if (ddbox->tric_dir[dim])
2722         {
2723             bfac *= ddbox->skew_fac[dim];
2724         }
2725         if ((comm->cell_f1[d] - comm->cell_f_max0[d])*bfac <  limit ||
2726             (comm->cell_f0[d] - comm->cell_f_min1[d])*bfac > -limit)
2727         {
2728             bInvalid = TRUE;
2729
2730             if (bFatal)
2731             {
2732                 char buf[22];
2733
2734                 /* This error should never be triggered under normal
2735                  * circumstances, but you never know ...
2736                  */
2737                 gmx_fatal(FARGS,"Step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with less nodes might avoid this issue.",
2738                           gmx_step_str(step,buf),
2739                           dim2char(dim),dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
2740             }
2741         }
2742     }
2743
2744     return bInvalid;
2745 }
2746
2747 static int dd_load_count(gmx_domdec_comm_t *comm)
2748 {
2749     return (comm->eFlop ? comm->flop_n : comm->cycl_n[ddCyclF]);
2750 }
2751
2752 static float dd_force_load(gmx_domdec_comm_t *comm)
2753 {
2754     float load;
2755
2756     if (comm->eFlop)
2757     {
2758         load = comm->flop;
2759         if (comm->eFlop > 1)
2760         {
2761             load *= 1.0 + (comm->eFlop - 1)*(0.1*rand()/RAND_MAX - 0.05);
2762         }
2763     }
2764     else
2765     {
2766         load = comm->cycl[ddCyclF];
2767         if (comm->cycl_n[ddCyclF] > 1)
2768         {
2769             /* Subtract the maximum of the last n cycle counts
2770              * to get rid of possible high counts due to other soures,
2771              * for instance system activity, that would otherwise
2772              * affect the dynamic load balancing.
2773              */
2774             load -= comm->cycl_max[ddCyclF];
2775         }
2776     }
2777
2778     return load;
2779 }
2780
2781 static void set_slb_pme_dim_f(gmx_domdec_t *dd,int dim,real **dim_f)
2782 {
2783     gmx_domdec_comm_t *comm;
2784     int i;
2785
2786     comm = dd->comm;
2787
2788     snew(*dim_f,dd->nc[dim]+1);
2789     (*dim_f)[0] = 0;
2790     for(i=1; i<dd->nc[dim]; i++)
2791     {
2792         if (comm->slb_frac[dim])
2793         {
2794             (*dim_f)[i] = (*dim_f)[i-1] + comm->slb_frac[dim][i-1];
2795         }
2796         else
2797         {
2798             (*dim_f)[i] = (real)i/(real)dd->nc[dim];
2799         }
2800     }
2801     (*dim_f)[dd->nc[dim]] = 1;
2802 }
2803
2804 static void init_ddpme(gmx_domdec_t *dd,gmx_ddpme_t *ddpme,int dimind)
2805 {
2806     int  pmeindex,slab,nso,i;
2807     ivec xyz;
2808
2809     if (dimind == 0 && dd->dim[0] == YY && dd->comm->npmenodes_x == 1)
2810     {
2811         ddpme->dim = YY;
2812     }
2813     else
2814     {
2815         ddpme->dim = dimind;
2816     }
2817     ddpme->dim_match = (ddpme->dim == dd->dim[dimind]);
2818
2819     ddpme->nslab = (ddpme->dim == 0 ?
2820                     dd->comm->npmenodes_x :
2821                     dd->comm->npmenodes_y);
2822
2823     if (ddpme->nslab <= 1)
2824     {
2825         return;
2826     }
2827
2828     nso = dd->comm->npmenodes/ddpme->nslab;
2829     /* Determine for each PME slab the PP location range for dimension dim */
2830     snew(ddpme->pp_min,ddpme->nslab);
2831     snew(ddpme->pp_max,ddpme->nslab);
2832     for(slab=0; slab<ddpme->nslab; slab++) {
2833         ddpme->pp_min[slab] = dd->nc[dd->dim[dimind]] - 1;
2834         ddpme->pp_max[slab] = 0;
2835     }
2836     for(i=0; i<dd->nnodes; i++) {
2837         ddindex2xyz(dd->nc,i,xyz);
2838         /* For y only use our y/z slab.
2839          * This assumes that the PME x grid size matches the DD grid size.
2840          */
2841         if (dimind == 0 || xyz[XX] == dd->ci[XX]) {
2842             pmeindex = ddindex2pmeindex(dd,i);
2843             if (dimind == 0) {
2844                 slab = pmeindex/nso;
2845             } else {
2846                 slab = pmeindex % ddpme->nslab;
2847             }
2848             ddpme->pp_min[slab] = min(ddpme->pp_min[slab],xyz[dimind]);
2849             ddpme->pp_max[slab] = max(ddpme->pp_max[slab],xyz[dimind]);
2850         }
2851     }
2852
2853     set_slb_pme_dim_f(dd,ddpme->dim,&ddpme->slb_dim_f);
2854 }
2855
2856 int dd_pme_maxshift_x(gmx_domdec_t *dd)
2857 {
2858     if (dd->comm->ddpme[0].dim == XX)
2859     {
2860         return dd->comm->ddpme[0].maxshift;
2861     }
2862     else
2863     {
2864         return 0;
2865     }
2866 }
2867
2868 int dd_pme_maxshift_y(gmx_domdec_t *dd)
2869 {
2870     if (dd->comm->ddpme[0].dim == YY)
2871     {
2872         return dd->comm->ddpme[0].maxshift;
2873     }
2874     else if (dd->comm->npmedecompdim >= 2 && dd->comm->ddpme[1].dim == YY)
2875     {
2876         return dd->comm->ddpme[1].maxshift;
2877     }
2878     else
2879     {
2880         return 0;
2881     }
2882 }
2883
2884 static void set_pme_maxshift(gmx_domdec_t *dd,gmx_ddpme_t *ddpme,
2885                              gmx_bool bUniform,gmx_ddbox_t *ddbox,real *cell_f)
2886 {
2887     gmx_domdec_comm_t *comm;
2888     int  nc,ns,s;
2889     int  *xmin,*xmax;
2890     real range,pme_boundary;
2891     int  sh;
2892
2893     comm = dd->comm;
2894     nc  = dd->nc[ddpme->dim];
2895     ns  = ddpme->nslab;
2896
2897     if (!ddpme->dim_match)
2898     {
2899         /* PP decomposition is not along dim: the worst situation */
2900         sh = ns/2;
2901     }
2902     else if (ns <= 3 || (bUniform && ns == nc))
2903     {
2904         /* The optimal situation */
2905         sh = 1;
2906     }
2907     else
2908     {
2909         /* We need to check for all pme nodes which nodes they
2910          * could possibly need to communicate with.
2911          */
2912         xmin = ddpme->pp_min;
2913         xmax = ddpme->pp_max;
2914         /* Allow for atoms to be maximally 2/3 times the cut-off
2915          * out of their DD cell. This is a reasonable balance between
2916          * between performance and support for most charge-group/cut-off
2917          * combinations.
2918          */
2919         range  = 2.0/3.0*comm->cutoff/ddbox->box_size[ddpme->dim];
2920         /* Avoid extra communication when we are exactly at a boundary */
2921         range *= 0.999;
2922
2923         sh = 1;
2924         for(s=0; s<ns; s++)
2925         {
2926             /* PME slab s spreads atoms between box frac. s/ns and (s+1)/ns */
2927             pme_boundary = (real)s/ns;
2928             while (sh+1 < ns &&
2929                    ((s-(sh+1) >= 0 &&
2930                      cell_f[xmax[s-(sh+1)   ]+1]     + range > pme_boundary) ||
2931                     (s-(sh+1) <  0 &&
2932                      cell_f[xmax[s-(sh+1)+ns]+1] - 1 + range > pme_boundary)))
2933             {
2934                 sh++;
2935             }
2936             pme_boundary = (real)(s+1)/ns;
2937             while (sh+1 < ns &&
2938                    ((s+(sh+1) <  ns &&
2939                      cell_f[xmin[s+(sh+1)   ]  ]     - range < pme_boundary) ||
2940                     (s+(sh+1) >= ns &&
2941                      cell_f[xmin[s+(sh+1)-ns]  ] + 1 - range < pme_boundary)))
2942             {
2943                 sh++;
2944             }
2945         }
2946     }
2947
2948     ddpme->maxshift = sh;
2949
2950     if (debug)
2951     {
2952         fprintf(debug,"PME slab communication range for dim %d is %d\n",
2953                 ddpme->dim,ddpme->maxshift);
2954     }
2955 }
2956
2957 static void check_box_size(gmx_domdec_t *dd,gmx_ddbox_t *ddbox)
2958 {
2959     int d,dim;
2960
2961     for(d=0; d<dd->ndim; d++)
2962     {
2963         dim = dd->dim[d];
2964         if (dim < ddbox->nboundeddim &&
2965             ddbox->box_size[dim]*ddbox->skew_fac[dim] <
2966             dd->nc[dim]*dd->comm->cellsize_limit*DD_CELL_MARGIN)
2967         {
2968             gmx_fatal(FARGS,"The %c-size of the box (%f) times the triclinic skew factor (%f) is smaller than the number of DD cells (%d) times the smallest allowed cell size (%f)\n",
2969                       dim2char(dim),ddbox->box_size[dim],ddbox->skew_fac[dim],
2970                       dd->nc[dim],dd->comm->cellsize_limit);
2971         }
2972     }
2973 }
2974
2975 static void set_dd_cell_sizes_slb(gmx_domdec_t *dd,gmx_ddbox_t *ddbox,
2976                                   gmx_bool bMaster,ivec npulse)
2977 {
2978     gmx_domdec_comm_t *comm;
2979     int  d,j;
2980     rvec cellsize_min;
2981     real *cell_x,cell_dx,cellsize;
2982
2983     comm = dd->comm;
2984
2985     for(d=0; d<DIM; d++)
2986     {
2987         cellsize_min[d] = ddbox->box_size[d]*ddbox->skew_fac[d];
2988         npulse[d] = 1;
2989         if (dd->nc[d] == 1 || comm->slb_frac[d] == NULL)
2990         {
2991             /* Uniform grid */
2992             cell_dx = ddbox->box_size[d]/dd->nc[d];
2993             if (bMaster)
2994             {
2995                 for(j=0; j<dd->nc[d]+1; j++)
2996                 {
2997                     dd->ma->cell_x[d][j] = ddbox->box0[d] + j*cell_dx;
2998                 }
2999             }
3000             else
3001             {
3002                 comm->cell_x0[d] = ddbox->box0[d] + (dd->ci[d]  )*cell_dx;
3003                 comm->cell_x1[d] = ddbox->box0[d] + (dd->ci[d]+1)*cell_dx;
3004             }
3005             cellsize = cell_dx*ddbox->skew_fac[d];
3006             while (cellsize*npulse[d] < comm->cutoff && npulse[d] < dd->nc[d]-1)
3007             {
3008                 npulse[d]++;
3009             }
3010             cellsize_min[d] = cellsize;
3011         }
3012         else
3013         {
3014             /* Statically load balanced grid */
3015             /* Also when we are not doing a master distribution we determine
3016              * all cell borders in a loop to obtain identical values
3017              * to the master distribution case and to determine npulse.
3018              */
3019             if (bMaster)
3020             {
3021                 cell_x = dd->ma->cell_x[d];
3022             }
3023             else
3024             {
3025                 snew(cell_x,dd->nc[d]+1);
3026             }
3027             cell_x[0] = ddbox->box0[d];
3028             for(j=0; j<dd->nc[d]; j++)
3029             {
3030                 cell_dx = ddbox->box_size[d]*comm->slb_frac[d][j];
3031                 cell_x[j+1] = cell_x[j] + cell_dx;
3032                 cellsize = cell_dx*ddbox->skew_fac[d];
3033                 while (cellsize*npulse[d] < comm->cutoff &&
3034                        npulse[d] < dd->nc[d]-1)
3035                 {
3036                     npulse[d]++;
3037                 }
3038                 cellsize_min[d] = min(cellsize_min[d],cellsize);
3039             }
3040             if (!bMaster)
3041             {
3042                 comm->cell_x0[d] = cell_x[dd->ci[d]];
3043                 comm->cell_x1[d] = cell_x[dd->ci[d]+1];
3044                 sfree(cell_x);
3045             }
3046         }
3047         /* The following limitation is to avoid that a cell would receive
3048          * some of its own home charge groups back over the periodic boundary.
3049          * Double charge groups cause trouble with the global indices.
3050          */
3051         if (d < ddbox->npbcdim &&
3052             dd->nc[d] > 1 && npulse[d] >= dd->nc[d])
3053         {
3054             gmx_fatal_collective(FARGS,NULL,dd,
3055                                  "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
3056                                  dim2char(d),ddbox->box_size[d],ddbox->skew_fac[d],
3057                                  comm->cutoff,
3058                                  dd->nc[d],dd->nc[d],
3059                                  dd->nnodes > dd->nc[d] ? "cells" : "processors");
3060         }
3061     }
3062
3063     if (!comm->bDynLoadBal)
3064     {
3065         copy_rvec(cellsize_min,comm->cellsize_min);
3066     }
3067
3068     for(d=0; d<comm->npmedecompdim; d++)
3069     {
3070         set_pme_maxshift(dd,&comm->ddpme[d],
3071                          comm->slb_frac[dd->dim[d]]==NULL,ddbox,
3072                          comm->ddpme[d].slb_dim_f);
3073     }
3074 }
3075
3076
3077 static void dd_cell_sizes_dlb_root_enforce_limits(gmx_domdec_t *dd,
3078                                        int d,int dim,gmx_domdec_root_t *root,
3079                                        gmx_ddbox_t *ddbox,
3080                                        gmx_bool bUniform,gmx_large_int_t step, real cellsize_limit_f, int range[])
3081 {
3082     gmx_domdec_comm_t *comm;
3083     int  ncd,i,j,nmin,nmin_old;
3084     gmx_bool bLimLo,bLimHi;
3085     real *cell_size;
3086     real fac,halfway,cellsize_limit_f_i,region_size;
3087     gmx_bool bPBC,bLastHi=FALSE;
3088     int nrange[]={range[0],range[1]};
3089
3090     region_size= root->cell_f[range[1]]-root->cell_f[range[0]];
3091
3092     comm = dd->comm;
3093
3094     ncd = dd->nc[dim];
3095
3096     bPBC = (dim < ddbox->npbcdim);
3097
3098     cell_size = root->buf_ncd;
3099
3100     if (debug)
3101     {
3102         fprintf(debug,"enforce_limits: %d %d\n",range[0],range[1]);
3103     }
3104
3105     /* First we need to check if the scaling does not make cells
3106      * smaller than the smallest allowed size.
3107      * We need to do this iteratively, since if a cell is too small,
3108      * it needs to be enlarged, which makes all the other cells smaller,
3109      * which could in turn make another cell smaller than allowed.
3110      */
3111     for(i=range[0]; i<range[1]; i++)
3112     {
3113         root->bCellMin[i] = FALSE;
3114     }
3115     nmin = 0;
3116     do
3117     {
3118         nmin_old = nmin;
3119         /* We need the total for normalization */
3120         fac = 0;
3121         for(i=range[0]; i<range[1]; i++)
3122         {
3123             if (root->bCellMin[i] == FALSE)
3124             {
3125                 fac += cell_size[i];
3126             }
3127         }
3128         fac = ( region_size - nmin*cellsize_limit_f)/fac; /* substracting cells already set to cellsize_limit_f */
3129         /* Determine the cell boundaries */
3130         for(i=range[0]; i<range[1]; i++)
3131         {
3132             if (root->bCellMin[i] == FALSE)
3133             {
3134                 cell_size[i] *= fac;
3135                 if (!bPBC && (i == 0 || i == dd->nc[dim] -1))
3136                 {
3137                     cellsize_limit_f_i = 0;
3138                 }
3139                 else
3140                 {
3141                     cellsize_limit_f_i = cellsize_limit_f;
3142                 }
3143                 if (cell_size[i] < cellsize_limit_f_i)
3144                 {
3145                     root->bCellMin[i] = TRUE;
3146                     cell_size[i] = cellsize_limit_f_i;
3147                     nmin++;
3148                 }
3149             }
3150             root->cell_f[i+1] = root->cell_f[i] + cell_size[i];
3151         }
3152     }
3153     while (nmin > nmin_old);
3154
3155     i=range[1]-1;
3156     cell_size[i] = root->cell_f[i+1] - root->cell_f[i];
3157     /* For this check we should not use DD_CELL_MARGIN,
3158      * but a slightly smaller factor,
3159      * since rounding could get use below the limit.
3160      */
3161     if (bPBC && cell_size[i] < cellsize_limit_f*DD_CELL_MARGIN2/DD_CELL_MARGIN)
3162     {
3163         char buf[22];
3164         gmx_fatal(FARGS,"Step %s: the dynamic load balancing could not balance dimension %c: box size %f, triclinic skew factor %f, #cells %d, minimum cell size %f\n",
3165                   gmx_step_str(step,buf),
3166                   dim2char(dim),ddbox->box_size[dim],ddbox->skew_fac[dim],
3167                   ncd,comm->cellsize_min[dim]);
3168     }
3169
3170     root->bLimited = (nmin > 0) || (range[0]>0) || (range[1]<ncd);
3171
3172     if (!bUniform)
3173     {
3174         /* Check if the boundary did not displace more than halfway
3175          * each of the cells it bounds, as this could cause problems,
3176          * especially when the differences between cell sizes are large.
3177          * If changes are applied, they will not make cells smaller
3178          * than the cut-off, as we check all the boundaries which
3179          * might be affected by a change and if the old state was ok,
3180          * the cells will at most be shrunk back to their old size.
3181          */
3182         for(i=range[0]+1; i<range[1]; i++)
3183         {
3184             halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i-1]);
3185             if (root->cell_f[i] < halfway)
3186             {
3187                 root->cell_f[i] = halfway;
3188                 /* Check if the change also causes shifts of the next boundaries */
3189                 for(j=i+1; j<range[1]; j++)
3190                 {
3191                     if (root->cell_f[j] < root->cell_f[j-1] + cellsize_limit_f)
3192                         root->cell_f[j] =  root->cell_f[j-1] + cellsize_limit_f;
3193                 }
3194             }
3195             halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i+1]);
3196             if (root->cell_f[i] > halfway)
3197             {
3198                 root->cell_f[i] = halfway;
3199                 /* Check if the change also causes shifts of the next boundaries */
3200                 for(j=i-1; j>=range[0]+1; j--)
3201                 {
3202                     if (root->cell_f[j] > root->cell_f[j+1] - cellsize_limit_f)
3203                         root->cell_f[j] = root->cell_f[j+1] - cellsize_limit_f;
3204                 }
3205             }
3206         }
3207     }
3208
3209     /* nrange is defined as [lower, upper) range for new call to enforce_limits */
3210     /* find highest violation of LimLo (a) and the following violation of LimHi (thus the lowest following) (b)
3211      * then call enforce_limits for (oldb,a), (a,b). In the next step: (b,nexta). oldb and nexta can be the boundaries.
3212      * for a and b nrange is used */
3213     if (d > 0)
3214     {
3215         /* Take care of the staggering of the cell boundaries */
3216         if (bUniform)
3217         {
3218             for(i=range[0]; i<range[1]; i++)
3219             {
3220                 root->cell_f_max0[i] = root->cell_f[i];
3221                 root->cell_f_min1[i] = root->cell_f[i+1];
3222             }
3223         }
3224         else
3225         {
3226             for(i=range[0]+1; i<range[1]; i++)
3227             {
3228                 bLimLo = (root->cell_f[i] < root->bound_min[i]);
3229                 bLimHi = (root->cell_f[i] > root->bound_max[i]);
3230                 if (bLimLo && bLimHi)
3231                 {
3232                     /* Both limits violated, try the best we can */
3233                     /* For this case we split the original range (range) in two parts and care about the other limitiations in the next iteration. */
3234                     root->cell_f[i] = 0.5*(root->bound_min[i] + root->bound_max[i]);
3235                     nrange[0]=range[0];
3236                     nrange[1]=i;
3237                     dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3238
3239                     nrange[0]=i;
3240                     nrange[1]=range[1];
3241                     dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3242
3243                     return;
3244                 }
3245                 else if (bLimLo)
3246                 {
3247                     /* root->cell_f[i] = root->bound_min[i]; */
3248                     nrange[1]=i;  /* only store violation location. There could be a LimLo violation following with an higher index */
3249                     bLastHi=FALSE;
3250                 }
3251                 else if (bLimHi && !bLastHi)
3252                 {
3253                     bLastHi=TRUE;
3254                     if (nrange[1] < range[1])   /* found a LimLo before */
3255                     {
3256                         root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
3257                         dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3258                         nrange[0]=nrange[1];
3259                     }
3260                     root->cell_f[i] = root->bound_max[i];
3261                     nrange[1]=i;
3262                     dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3263                     nrange[0]=i;
3264                     nrange[1]=range[1];
3265                 }
3266             }
3267             if (nrange[1] < range[1])   /* found last a LimLo */
3268             {
3269                 root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
3270                 dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3271                 nrange[0]=nrange[1];
3272                 nrange[1]=range[1];
3273                 dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3274             }
3275             else if (nrange[0] > range[0]) /* found at least one LimHi */
3276             {
3277                 dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
3278             }
3279         }
3280     }
3281 }
3282
3283
3284 static void set_dd_cell_sizes_dlb_root(gmx_domdec_t *dd,
3285                                        int d,int dim,gmx_domdec_root_t *root,
3286                                        gmx_ddbox_t *ddbox,gmx_bool bDynamicBox,
3287                                        gmx_bool bUniform,gmx_large_int_t step)
3288 {
3289     gmx_domdec_comm_t *comm;
3290     int  ncd,d1,i,j,pos;
3291     real *cell_size;
3292     real load_aver,load_i,imbalance,change,change_max,sc;
3293     real cellsize_limit_f,dist_min_f,dist_min_f_hard,space;
3294     real change_limit;
3295     real relax = 0.5;
3296     gmx_bool bPBC;
3297     int range[] = { 0, 0 };
3298
3299     comm = dd->comm;
3300
3301     /* Convert the maximum change from the input percentage to a fraction */
3302     change_limit = comm->dlb_scale_lim*0.01;
3303
3304     ncd = dd->nc[dim];
3305
3306     bPBC = (dim < ddbox->npbcdim);
3307
3308     cell_size = root->buf_ncd;
3309
3310     /* Store the original boundaries */
3311     for(i=0; i<ncd+1; i++)
3312     {
3313         root->old_cell_f[i] = root->cell_f[i];
3314     }
3315     if (bUniform) {
3316         for(i=0; i<ncd; i++)
3317         {
3318             cell_size[i] = 1.0/ncd;
3319         }
3320     }
3321     else if (dd_load_count(comm))
3322     {
3323         load_aver = comm->load[d].sum_m/ncd;
3324         change_max = 0;
3325         for(i=0; i<ncd; i++)
3326         {
3327             /* Determine the relative imbalance of cell i */
3328             load_i = comm->load[d].load[i*comm->load[d].nload+2];
3329             imbalance = (load_i - load_aver)/(load_aver>0 ? load_aver : 1);
3330             /* Determine the change of the cell size using underrelaxation */
3331             change = -relax*imbalance;
3332             change_max = max(change_max,max(change,-change));
3333         }
3334         /* Limit the amount of scaling.
3335          * We need to use the same rescaling for all cells in one row,
3336          * otherwise the load balancing might not converge.
3337          */
3338         sc = relax;
3339         if (change_max > change_limit)
3340         {
3341             sc *= change_limit/change_max;
3342         }
3343         for(i=0; i<ncd; i++)
3344         {
3345             /* Determine the relative imbalance of cell i */
3346             load_i = comm->load[d].load[i*comm->load[d].nload+2];
3347             imbalance = (load_i - load_aver)/(load_aver>0 ? load_aver : 1);
3348             /* Determine the change of the cell size using underrelaxation */
3349             change = -sc*imbalance;
3350             cell_size[i] = (root->cell_f[i+1]-root->cell_f[i])*(1 + change);
3351         }
3352     }
3353
3354     cellsize_limit_f  = comm->cellsize_min[dim]/ddbox->box_size[dim];
3355     cellsize_limit_f *= DD_CELL_MARGIN;
3356     dist_min_f_hard   = grid_jump_limit(comm,comm->cutoff,d)/ddbox->box_size[dim];
3357     dist_min_f        = dist_min_f_hard * DD_CELL_MARGIN;
3358     if (ddbox->tric_dir[dim])
3359     {
3360         cellsize_limit_f /= ddbox->skew_fac[dim];
3361         dist_min_f       /= ddbox->skew_fac[dim];
3362     }
3363     if (bDynamicBox && d > 0)
3364     {
3365         dist_min_f *= DD_PRES_SCALE_MARGIN;
3366     }
3367     if (d > 0 && !bUniform)
3368     {
3369         /* Make sure that the grid is not shifted too much */
3370         for(i=1; i<ncd; i++) {
3371             if (root->cell_f_min1[i] - root->cell_f_max0[i-1] < 2 * dist_min_f_hard)
3372             {
3373                 gmx_incons("Inconsistent DD boundary staggering limits!");
3374             }
3375             root->bound_min[i] = root->cell_f_max0[i-1] + dist_min_f;
3376             space = root->cell_f[i] - (root->cell_f_max0[i-1] + dist_min_f);
3377             if (space > 0) {
3378                 root->bound_min[i] += 0.5*space;
3379             }
3380             root->bound_max[i] = root->cell_f_min1[i] - dist_min_f;
3381             space = root->cell_f[i] - (root->cell_f_min1[i] - dist_min_f);
3382             if (space < 0) {
3383                 root->bound_max[i] += 0.5*space;
3384             }
3385             if (debug)
3386             {
3387                 fprintf(debug,
3388                         "dim %d boundary %d %.3f < %.3f < %.3f < %.3f < %.3f\n",
3389                         d,i,
3390                         root->cell_f_max0[i-1] + dist_min_f,
3391                         root->bound_min[i],root->cell_f[i],root->bound_max[i],
3392                         root->cell_f_min1[i] - dist_min_f);
3393             }
3394         }
3395     }
3396     range[1]=ncd;
3397     root->cell_f[0] = 0;
3398     root->cell_f[ncd] = 1;
3399     dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, range);
3400
3401
3402     /* After the checks above, the cells should obey the cut-off
3403      * restrictions, but it does not hurt to check.
3404      */
3405     for(i=0; i<ncd; i++)
3406     {
3407         if (debug)
3408         {
3409             fprintf(debug,"Relative bounds dim %d  cell %d: %f %f\n",
3410                     dim,i,root->cell_f[i],root->cell_f[i+1]);
3411         }
3412
3413         if ((bPBC || (i != 0 && i != dd->nc[dim]-1)) &&
3414             root->cell_f[i+1] - root->cell_f[i] <
3415             cellsize_limit_f/DD_CELL_MARGIN)
3416         {
3417             char buf[22];
3418             fprintf(stderr,
3419                     "\nWARNING step %s: direction %c, cell %d too small: %f\n",
3420                     gmx_step_str(step,buf),dim2char(dim),i,
3421                     (root->cell_f[i+1] - root->cell_f[i])
3422                     *ddbox->box_size[dim]*ddbox->skew_fac[dim]);
3423         }
3424     }
3425
3426     pos = ncd + 1;
3427     /* Store the cell boundaries of the lower dimensions at the end */
3428     for(d1=0; d1<d; d1++)
3429     {
3430         root->cell_f[pos++] = comm->cell_f0[d1];
3431         root->cell_f[pos++] = comm->cell_f1[d1];
3432     }
3433
3434     if (d < comm->npmedecompdim)
3435     {
3436         /* The master determines the maximum shift for
3437          * the coordinate communication between separate PME nodes.
3438          */
3439         set_pme_maxshift(dd,&comm->ddpme[d],bUniform,ddbox,root->cell_f);
3440     }
3441     root->cell_f[pos++] = comm->ddpme[0].maxshift;
3442     if (d >= 1)
3443     {
3444         root->cell_f[pos++] = comm->ddpme[1].maxshift;
3445     }
3446 }
3447
3448 static void relative_to_absolute_cell_bounds(gmx_domdec_t *dd,
3449                                              gmx_ddbox_t *ddbox,int dimind)
3450 {
3451     gmx_domdec_comm_t *comm;
3452     int dim;
3453
3454     comm = dd->comm;
3455
3456     /* Set the cell dimensions */
3457     dim = dd->dim[dimind];
3458     comm->cell_x0[dim] = comm->cell_f0[dimind]*ddbox->box_size[dim];
3459     comm->cell_x1[dim] = comm->cell_f1[dimind]*ddbox->box_size[dim];
3460     if (dim >= ddbox->nboundeddim)
3461     {
3462         comm->cell_x0[dim] += ddbox->box0[dim];
3463         comm->cell_x1[dim] += ddbox->box0[dim];
3464     }
3465 }
3466
3467 static void distribute_dd_cell_sizes_dlb(gmx_domdec_t *dd,
3468                                          int d,int dim,real *cell_f_row,
3469                                          gmx_ddbox_t *ddbox)
3470 {
3471     gmx_domdec_comm_t *comm;
3472     int d1,dim1,pos;
3473
3474     comm = dd->comm;
3475
3476 #ifdef GMX_MPI
3477     /* Each node would only need to know two fractions,
3478      * but it is probably cheaper to broadcast the whole array.
3479      */
3480     MPI_Bcast(cell_f_row,DD_CELL_F_SIZE(dd,d)*sizeof(real),MPI_BYTE,
3481               0,comm->mpi_comm_load[d]);
3482 #endif
3483     /* Copy the fractions for this dimension from the buffer */
3484     comm->cell_f0[d] = cell_f_row[dd->ci[dim]  ];
3485     comm->cell_f1[d] = cell_f_row[dd->ci[dim]+1];
3486     /* The whole array was communicated, so set the buffer position */
3487     pos = dd->nc[dim] + 1;
3488     for(d1=0; d1<=d; d1++)
3489     {
3490         if (d1 < d)
3491         {
3492             /* Copy the cell fractions of the lower dimensions */
3493             comm->cell_f0[d1] = cell_f_row[pos++];
3494             comm->cell_f1[d1] = cell_f_row[pos++];
3495         }
3496         relative_to_absolute_cell_bounds(dd,ddbox,d1);
3497     }
3498     /* Convert the communicated shift from float to int */
3499     comm->ddpme[0].maxshift = (int)(cell_f_row[pos++] + 0.5);
3500     if (d >= 1)
3501     {
3502         comm->ddpme[1].maxshift = (int)(cell_f_row[pos++] + 0.5);
3503     }
3504 }
3505
3506 static void set_dd_cell_sizes_dlb_change(gmx_domdec_t *dd,
3507                                          gmx_ddbox_t *ddbox,gmx_bool bDynamicBox,
3508                                          gmx_bool bUniform,gmx_large_int_t step)
3509 {
3510     gmx_domdec_comm_t *comm;
3511     int d,dim,d1;
3512     gmx_bool bRowMember,bRowRoot;
3513     real *cell_f_row;
3514
3515     comm = dd->comm;
3516
3517     for(d=0; d<dd->ndim; d++)
3518     {
3519         dim = dd->dim[d];
3520         bRowMember = TRUE;
3521         bRowRoot = TRUE;
3522         for(d1=d; d1<dd->ndim; d1++)
3523         {
3524             if (dd->ci[dd->dim[d1]] > 0)
3525             {
3526                 if (d1 > d)
3527                 {
3528                     bRowMember = FALSE;
3529                 }
3530                 bRowRoot = FALSE;
3531             }
3532         }
3533         if (bRowMember)
3534         {
3535             if (bRowRoot)
3536             {
3537                 set_dd_cell_sizes_dlb_root(dd,d,dim,comm->root[d],
3538                                            ddbox,bDynamicBox,bUniform,step);
3539                 cell_f_row = comm->root[d]->cell_f;
3540             }
3541             else
3542             {
3543                 cell_f_row = comm->cell_f_row;
3544             }
3545             distribute_dd_cell_sizes_dlb(dd,d,dim,cell_f_row,ddbox);
3546         }
3547     }
3548 }
3549
3550 static void set_dd_cell_sizes_dlb_nochange(gmx_domdec_t *dd,gmx_ddbox_t *ddbox)
3551 {
3552     int d;
3553
3554     /* This function assumes the box is static and should therefore
3555      * not be called when the box has changed since the last
3556      * call to dd_partition_system.
3557      */
3558     for(d=0; d<dd->ndim; d++)
3559     {
3560         relative_to_absolute_cell_bounds(dd,ddbox,d);
3561     }
3562 }
3563
3564
3565
3566 static void set_dd_cell_sizes_dlb(gmx_domdec_t *dd,
3567                                   gmx_ddbox_t *ddbox,gmx_bool bDynamicBox,
3568                                   gmx_bool bUniform,gmx_bool bDoDLB,gmx_large_int_t step,
3569                                   gmx_wallcycle_t wcycle)
3570 {
3571     gmx_domdec_comm_t *comm;
3572     int dim;
3573
3574     comm = dd->comm;
3575
3576     if (bDoDLB)
3577     {
3578         wallcycle_start(wcycle,ewcDDCOMMBOUND);
3579         set_dd_cell_sizes_dlb_change(dd,ddbox,bDynamicBox,bUniform,step);
3580         wallcycle_stop(wcycle,ewcDDCOMMBOUND);
3581     }
3582     else if (bDynamicBox)
3583     {
3584         set_dd_cell_sizes_dlb_nochange(dd,ddbox);
3585     }
3586
3587     /* Set the dimensions for which no DD is used */
3588     for(dim=0; dim<DIM; dim++) {
3589         if (dd->nc[dim] == 1) {
3590             comm->cell_x0[dim] = 0;
3591             comm->cell_x1[dim] = ddbox->box_size[dim];
3592             if (dim >= ddbox->nboundeddim)
3593             {
3594                 comm->cell_x0[dim] += ddbox->box0[dim];
3595                 comm->cell_x1[dim] += ddbox->box0[dim];
3596             }
3597         }
3598     }
3599 }
3600
3601 static void realloc_comm_ind(gmx_domdec_t *dd,ivec npulse)
3602 {
3603     int d,np,i;
3604     gmx_domdec_comm_dim_t *cd;
3605
3606     for(d=0; d<dd->ndim; d++)
3607     {
3608         cd = &dd->comm->cd[d];
3609         np = npulse[dd->dim[d]];
3610         if (np > cd->np_nalloc)
3611         {
3612             if (debug)
3613             {
3614                 fprintf(debug,"(Re)allocing cd for %c to %d pulses\n",
3615                         dim2char(dd->dim[d]),np);
3616             }
3617             if (DDMASTER(dd) && cd->np_nalloc > 0)
3618             {
3619                 fprintf(stderr,"\nIncreasing the number of cell to communicate in dimension %c to %d for the first time\n",dim2char(dd->dim[d]),np);
3620             }
3621             srenew(cd->ind,np);
3622             for(i=cd->np_nalloc; i<np; i++)
3623             {
3624                 cd->ind[i].index  = NULL;
3625                 cd->ind[i].nalloc = 0;
3626             }
3627             cd->np_nalloc = np;
3628         }
3629         cd->np = np;
3630     }
3631 }
3632
3633
3634 static void set_dd_cell_sizes(gmx_domdec_t *dd,
3635                               gmx_ddbox_t *ddbox,gmx_bool bDynamicBox,
3636                               gmx_bool bUniform,gmx_bool bDoDLB,gmx_large_int_t step,
3637                               gmx_wallcycle_t wcycle)
3638 {
3639     gmx_domdec_comm_t *comm;
3640     int  d;
3641     ivec npulse;
3642
3643     comm = dd->comm;
3644
3645     /* Copy the old cell boundaries for the cg displacement check */
3646     copy_rvec(comm->cell_x0,comm->old_cell_x0);
3647     copy_rvec(comm->cell_x1,comm->old_cell_x1);
3648
3649     if (comm->bDynLoadBal)
3650     {
3651         if (DDMASTER(dd))
3652         {
3653             check_box_size(dd,ddbox);
3654         }
3655         set_dd_cell_sizes_dlb(dd,ddbox,bDynamicBox,bUniform,bDoDLB,step,wcycle);
3656     }
3657     else
3658     {
3659         set_dd_cell_sizes_slb(dd,ddbox,FALSE,npulse);
3660         realloc_comm_ind(dd,npulse);
3661     }
3662
3663     if (debug)
3664     {
3665         for(d=0; d<DIM; d++)
3666         {
3667             fprintf(debug,"cell_x[%d] %f - %f skew_fac %f\n",
3668                     d,comm->cell_x0[d],comm->cell_x1[d],ddbox->skew_fac[d]);
3669         }
3670     }
3671 }
3672
3673 static void comm_dd_ns_cell_sizes(gmx_domdec_t *dd,
3674                                   gmx_ddbox_t *ddbox,
3675                                   rvec cell_ns_x0,rvec cell_ns_x1,
3676                                   gmx_large_int_t step)
3677 {
3678     gmx_domdec_comm_t *comm;
3679     int dim_ind,dim;
3680
3681     comm = dd->comm;
3682
3683     for(dim_ind=0; dim_ind<dd->ndim; dim_ind++)
3684     {
3685         dim = dd->dim[dim_ind];
3686
3687         /* Without PBC we don't have restrictions on the outer cells */
3688         if (!(dim >= ddbox->npbcdim &&
3689               (dd->ci[dim] == 0 || dd->ci[dim] == dd->nc[dim] - 1)) &&
3690             comm->bDynLoadBal &&
3691             (comm->cell_x1[dim] - comm->cell_x0[dim])*ddbox->skew_fac[dim] <
3692             comm->cellsize_min[dim])
3693         {
3694             char buf[22];
3695             gmx_fatal(FARGS,"Step %s: The %c-size (%f) times the triclinic skew factor (%f) is smaller than the smallest allowed cell size (%f) for domain decomposition grid cell %d %d %d",
3696                       gmx_step_str(step,buf),dim2char(dim),
3697                       comm->cell_x1[dim] - comm->cell_x0[dim],
3698                       ddbox->skew_fac[dim],
3699                       dd->comm->cellsize_min[dim],
3700                       dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
3701         }
3702     }
3703
3704     if ((dd->bGridJump && dd->ndim > 1) || ddbox->nboundeddim < DIM)
3705     {
3706         /* Communicate the boundaries and update cell_ns_x0/1 */
3707         dd_move_cellx(dd,ddbox,cell_ns_x0,cell_ns_x1);
3708         if (dd->bGridJump && dd->ndim > 1)
3709         {
3710             check_grid_jump(step,dd,dd->comm->cutoff,ddbox,TRUE);
3711         }
3712     }
3713 }
3714
3715 static void make_tric_corr_matrix(int npbcdim,matrix box,matrix tcm)
3716 {
3717     if (YY < npbcdim)
3718     {
3719         tcm[YY][XX] = -box[YY][XX]/box[YY][YY];
3720     }
3721     else
3722     {
3723         tcm[YY][XX] = 0;
3724     }
3725     if (ZZ < npbcdim)
3726     {
3727         tcm[ZZ][XX] = -(box[ZZ][YY]*tcm[YY][XX] + box[ZZ][XX])/box[ZZ][ZZ];
3728         tcm[ZZ][YY] = -box[ZZ][YY]/box[ZZ][ZZ];
3729     }
3730     else
3731     {
3732         tcm[ZZ][XX] = 0;
3733         tcm[ZZ][YY] = 0;
3734     }
3735 }
3736
3737 static void check_screw_box(matrix box)
3738 {
3739     /* Mathematical limitation */
3740     if (box[YY][XX] != 0 || box[ZZ][XX] != 0)
3741     {
3742         gmx_fatal(FARGS,"With screw pbc the unit cell can not have non-zero off-diagonal x-components");
3743     }
3744
3745     /* Limitation due to the asymmetry of the eighth shell method */
3746     if (box[ZZ][YY] != 0)
3747     {
3748         gmx_fatal(FARGS,"pbc=screw with non-zero box_zy is not supported");
3749     }
3750 }
3751
3752 static void distribute_cg(FILE *fplog,gmx_large_int_t step,
3753                           matrix box,ivec tric_dir,t_block *cgs,rvec pos[],
3754                           gmx_domdec_t *dd)
3755 {
3756     gmx_domdec_master_t *ma;
3757     int **tmp_ind=NULL,*tmp_nalloc=NULL;
3758     int  i,icg,j,k,k0,k1,d,npbcdim;
3759     matrix tcm;
3760     rvec box_size,cg_cm;
3761     ivec ind;
3762     real nrcg,inv_ncg,pos_d;
3763     atom_id *cgindex;
3764     gmx_bool bUnbounded,bScrew;
3765
3766     ma = dd->ma;
3767
3768     if (tmp_ind == NULL)
3769     {
3770         snew(tmp_nalloc,dd->nnodes);
3771         snew(tmp_ind,dd->nnodes);
3772         for(i=0; i<dd->nnodes; i++)
3773         {
3774             tmp_nalloc[i] = over_alloc_large(cgs->nr/dd->nnodes+1);
3775             snew(tmp_ind[i],tmp_nalloc[i]);
3776         }
3777     }
3778
3779     /* Clear the count */
3780     for(i=0; i<dd->nnodes; i++)
3781     {
3782         ma->ncg[i] = 0;
3783         ma->nat[i] = 0;
3784     }
3785
3786     make_tric_corr_matrix(dd->npbcdim,box,tcm);
3787
3788     cgindex = cgs->index;
3789
3790     /* Compute the center of geometry for all charge groups */
3791     for(icg=0; icg<cgs->nr; icg++)
3792     {
3793         k0      = cgindex[icg];
3794         k1      = cgindex[icg+1];
3795         nrcg    = k1 - k0;
3796         if (nrcg == 1)
3797         {
3798             copy_rvec(pos[k0],cg_cm);
3799         }
3800         else
3801         {
3802             inv_ncg = 1.0/nrcg;
3803
3804             clear_rvec(cg_cm);
3805             for(k=k0; (k<k1); k++)
3806             {
3807                 rvec_inc(cg_cm,pos[k]);
3808             }
3809             for(d=0; (d<DIM); d++)
3810             {
3811                 cg_cm[d] *= inv_ncg;
3812             }
3813         }
3814         /* Put the charge group in the box and determine the cell index */
3815         for(d=DIM-1; d>=0; d--) {
3816             pos_d = cg_cm[d];
3817             if (d < dd->npbcdim)
3818             {
3819                 bScrew = (dd->bScrewPBC && d == XX);
3820                 if (tric_dir[d] && dd->nc[d] > 1)
3821                 {
3822                     /* Use triclinic coordintates for this dimension */
3823                     for(j=d+1; j<DIM; j++)
3824                     {
3825                         pos_d += cg_cm[j]*tcm[j][d];
3826                     }
3827                 }
3828                 while(pos_d >= box[d][d])
3829                 {
3830                     pos_d -= box[d][d];
3831                     rvec_dec(cg_cm,box[d]);
3832                     if (bScrew)
3833                     {
3834                         cg_cm[YY] = box[YY][YY] - cg_cm[YY];
3835                         cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
3836                     }
3837                     for(k=k0; (k<k1); k++)
3838                     {
3839                         rvec_dec(pos[k],box[d]);
3840                         if (bScrew)
3841                         {
3842                             pos[k][YY] = box[YY][YY] - pos[k][YY];
3843                             pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
3844                         }
3845                     }
3846                 }
3847                 while(pos_d < 0)
3848                 {
3849                     pos_d += box[d][d];
3850                     rvec_inc(cg_cm,box[d]);
3851                     if (bScrew)
3852                     {
3853                         cg_cm[YY] = box[YY][YY] - cg_cm[YY];
3854                         cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
3855                     }
3856                     for(k=k0; (k<k1); k++)
3857                     {
3858                         rvec_inc(pos[k],box[d]);
3859                         if (bScrew) {
3860                             pos[k][YY] = box[YY][YY] - pos[k][YY];
3861                             pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
3862                         }
3863                     }
3864                 }
3865             }
3866             /* This could be done more efficiently */
3867             ind[d] = 0;
3868             while(ind[d]+1 < dd->nc[d] && pos_d >= ma->cell_x[d][ind[d]+1])
3869             {
3870                 ind[d]++;
3871             }
3872         }
3873         i = dd_index(dd->nc,ind);
3874         if (ma->ncg[i] == tmp_nalloc[i])
3875         {
3876             tmp_nalloc[i] = over_alloc_large(ma->ncg[i]+1);
3877             srenew(tmp_ind[i],tmp_nalloc[i]);
3878         }
3879         tmp_ind[i][ma->ncg[i]] = icg;
3880         ma->ncg[i]++;
3881         ma->nat[i] += cgindex[icg+1] - cgindex[icg];
3882     }
3883
3884     k1 = 0;
3885     for(i=0; i<dd->nnodes; i++)
3886     {
3887         ma->index[i] = k1;
3888         for(k=0; k<ma->ncg[i]; k++)
3889         {
3890             ma->cg[k1++] = tmp_ind[i][k];
3891         }
3892     }
3893     ma->index[dd->nnodes] = k1;
3894
3895     for(i=0; i<dd->nnodes; i++)
3896     {
3897         sfree(tmp_ind[i]);
3898     }
3899     sfree(tmp_ind);
3900     sfree(tmp_nalloc);
3901
3902     if (fplog)
3903     {
3904         char buf[22];
3905         fprintf(fplog,"Charge group distribution at step %s:",
3906                 gmx_step_str(step,buf));
3907         for(i=0; i<dd->nnodes; i++)
3908         {
3909             fprintf(fplog," %d",ma->ncg[i]);
3910         }
3911         fprintf(fplog,"\n");
3912     }
3913 }
3914
3915 static void get_cg_distribution(FILE *fplog,gmx_large_int_t step,gmx_domdec_t *dd,
3916                                 t_block *cgs,matrix box,gmx_ddbox_t *ddbox,
3917                                 rvec pos[])
3918 {
3919     gmx_domdec_master_t *ma=NULL;
3920     ivec npulse;
3921     int  i,cg_gl;
3922     int  *ibuf,buf2[2] = { 0, 0 };
3923     gmx_bool bMaster = DDMASTER(dd);
3924     if (bMaster)
3925     {
3926         ma = dd->ma;
3927
3928         if (dd->bScrewPBC)
3929         {
3930             check_screw_box(box);
3931         }
3932
3933         set_dd_cell_sizes_slb(dd,ddbox,TRUE,npulse);
3934
3935         distribute_cg(fplog,step,box,ddbox->tric_dir,cgs,pos,dd);
3936         for(i=0; i<dd->nnodes; i++)
3937         {
3938             ma->ibuf[2*i]   = ma->ncg[i];
3939             ma->ibuf[2*i+1] = ma->nat[i];
3940         }
3941         ibuf = ma->ibuf;
3942     }
3943     else
3944     {
3945         ibuf = NULL;
3946     }
3947     dd_scatter(dd,2*sizeof(int),ibuf,buf2);
3948
3949     dd->ncg_home = buf2[0];
3950     dd->nat_home = buf2[1];
3951     dd->ncg_tot  = dd->ncg_home;
3952     dd->nat_tot  = dd->nat_home;
3953     if (dd->ncg_home > dd->cg_nalloc || dd->cg_nalloc == 0)
3954     {
3955         dd->cg_nalloc = over_alloc_dd(dd->ncg_home);
3956         srenew(dd->index_gl,dd->cg_nalloc);
3957         srenew(dd->cgindex,dd->cg_nalloc+1);
3958     }
3959     if (bMaster)
3960     {
3961         for(i=0; i<dd->nnodes; i++)
3962         {
3963             ma->ibuf[i] = ma->ncg[i]*sizeof(int);
3964             ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
3965         }
3966     }
3967
3968     dd_scatterv(dd,
3969                 DDMASTER(dd) ? ma->ibuf : NULL,
3970                 DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
3971                 DDMASTER(dd) ? ma->cg : NULL,
3972                 dd->ncg_home*sizeof(int),dd->index_gl);
3973
3974     /* Determine the home charge group sizes */
3975     dd->cgindex[0] = 0;
3976     for(i=0; i<dd->ncg_home; i++)
3977     {
3978         cg_gl = dd->index_gl[i];
3979         dd->cgindex[i+1] =
3980             dd->cgindex[i] + cgs->index[cg_gl+1] - cgs->index[cg_gl];
3981     }
3982
3983     if (debug)
3984     {
3985         fprintf(debug,"Home charge groups:\n");
3986         for(i=0; i<dd->ncg_home; i++)
3987         {
3988             fprintf(debug," %d",dd->index_gl[i]);
3989             if (i % 10 == 9)
3990                 fprintf(debug,"\n");
3991         }
3992         fprintf(debug,"\n");
3993     }
3994 }
3995
3996 static int compact_and_copy_vec_at(int ncg,int *move,
3997                                    int *cgindex,
3998                                    int nvec,int vec,
3999                                    rvec *src,gmx_domdec_comm_t *comm,
4000                                    gmx_bool bCompact)
4001 {
4002     int m,icg,i,i0,i1,nrcg;
4003     int home_pos;
4004     int pos_vec[DIM*2];
4005
4006     home_pos = 0;
4007
4008     for(m=0; m<DIM*2; m++)
4009     {
4010         pos_vec[m] = 0;
4011     }
4012
4013     i0 = 0;
4014     for(icg=0; icg<ncg; icg++)
4015     {
4016         i1 = cgindex[icg+1];
4017         m = move[icg];
4018         if (m == -1)
4019         {
4020             if (bCompact)
4021             {
4022                 /* Compact the home array in place */
4023                 for(i=i0; i<i1; i++)
4024                 {
4025                     copy_rvec(src[i],src[home_pos++]);
4026                 }
4027             }
4028         }
4029         else
4030         {
4031             /* Copy to the communication buffer */
4032             nrcg = i1 - i0;
4033             pos_vec[m] += 1 + vec*nrcg;
4034             for(i=i0; i<i1; i++)
4035             {
4036                 copy_rvec(src[i],comm->cgcm_state[m][pos_vec[m]++]);
4037             }
4038             pos_vec[m] += (nvec - vec - 1)*nrcg;
4039         }
4040         if (!bCompact)
4041         {
4042             home_pos += i1 - i0;
4043         }
4044         i0 = i1;
4045     }
4046
4047     return home_pos;
4048 }
4049
4050 static int compact_and_copy_vec_cg(int ncg,int *move,
4051                                    int *cgindex,
4052                                    int nvec,rvec *src,gmx_domdec_comm_t *comm,
4053                                    gmx_bool bCompact)
4054 {
4055     int m,icg,i0,i1,nrcg;
4056     int home_pos;
4057     int pos_vec[DIM*2];
4058
4059     home_pos = 0;
4060
4061     for(m=0; m<DIM*2; m++)
4062     {
4063         pos_vec[m] = 0;
4064     }
4065
4066     i0 = 0;
4067     for(icg=0; icg<ncg; icg++)
4068     {
4069         i1 = cgindex[icg+1];
4070         m = move[icg];
4071         if (m == -1)
4072         {
4073             if (bCompact)
4074             {
4075                 /* Compact the home array in place */
4076                 copy_rvec(src[icg],src[home_pos++]);
4077             }
4078         }
4079         else
4080         {
4081             nrcg = i1 - i0;
4082             /* Copy to the communication buffer */
4083             copy_rvec(src[icg],comm->cgcm_state[m][pos_vec[m]]);
4084             pos_vec[m] += 1 + nrcg*nvec;
4085         }
4086         i0 = i1;
4087     }
4088     if (!bCompact)
4089     {
4090         home_pos = ncg;
4091     }
4092
4093     return home_pos;
4094 }
4095
4096 static int compact_ind(int ncg,int *move,
4097                        int *index_gl,int *cgindex,
4098                        int *gatindex,
4099                        gmx_ga2la_t ga2la,char *bLocalCG,
4100                        int *cginfo)
4101 {
4102     int cg,nat,a0,a1,a,a_gl;
4103     int home_pos;
4104
4105     home_pos = 0;
4106     nat = 0;
4107     for(cg=0; cg<ncg; cg++)
4108     {
4109         a0 = cgindex[cg];
4110         a1 = cgindex[cg+1];
4111         if (move[cg] == -1)
4112         {
4113             /* Compact the home arrays in place.
4114              * Anything that can be done here avoids access to global arrays.
4115              */
4116             cgindex[home_pos] = nat;
4117             for(a=a0; a<a1; a++)
4118             {
4119                 a_gl = gatindex[a];
4120                 gatindex[nat] = a_gl;
4121                 /* The cell number stays 0, so we don't need to set it */
4122                 ga2la_change_la(ga2la,a_gl,nat);
4123                 nat++;
4124             }
4125             index_gl[home_pos] = index_gl[cg];
4126             cginfo[home_pos]   = cginfo[cg];
4127             /* The charge group remains local, so bLocalCG does not change */
4128             home_pos++;
4129         }
4130         else
4131         {
4132             /* Clear the global indices */
4133             for(a=a0; a<a1; a++)
4134             {
4135                 ga2la_del(ga2la,gatindex[a]);
4136             }
4137             if (bLocalCG)
4138             {
4139                 bLocalCG[index_gl[cg]] = FALSE;
4140             }
4141         }
4142     }
4143     cgindex[home_pos] = nat;
4144
4145     return home_pos;
4146 }
4147
4148 static void clear_and_mark_ind(int ncg,int *move,
4149                                int *index_gl,int *cgindex,int *gatindex,
4150                                gmx_ga2la_t ga2la,char *bLocalCG,
4151                                int *cell_index)
4152 {
4153     int cg,a0,a1,a;
4154
4155     for(cg=0; cg<ncg; cg++)
4156     {
4157         if (move[cg] >= 0)
4158         {
4159             a0 = cgindex[cg];
4160             a1 = cgindex[cg+1];
4161             /* Clear the global indices */
4162             for(a=a0; a<a1; a++)
4163             {
4164                 ga2la_del(ga2la,gatindex[a]);
4165             }
4166             if (bLocalCG)
4167             {
4168                 bLocalCG[index_gl[cg]] = FALSE;
4169             }
4170             /* Signal that this cg has moved using the ns cell index.
4171              * Here we set it to -1. fill_grid will change it
4172              * from -1 to NSGRID_SIGNAL_MOVED_FAC*grid->ncells.
4173              */
4174             cell_index[cg] = -1;
4175         }
4176     }
4177 }
4178
4179 static void print_cg_move(FILE *fplog,
4180                           gmx_domdec_t *dd,
4181                           gmx_large_int_t step,int cg,int dim,int dir,
4182                           gmx_bool bHaveLimitdAndCMOld,real limitd,
4183                           rvec cm_old,rvec cm_new,real pos_d)
4184 {
4185     gmx_domdec_comm_t *comm;
4186     char buf[22];
4187
4188     comm = dd->comm;
4189
4190     fprintf(fplog,"\nStep %s:\n",gmx_step_str(step,buf));
4191     if (bHaveLimitdAndCMOld)
4192     {
4193         fprintf(fplog,"The charge group starting at atom %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
4194                 ddglatnr(dd,dd->cgindex[cg]),limitd,dim2char(dim));
4195     }
4196     else
4197     {
4198         fprintf(fplog,"The charge group starting at atom %d moved than the distance allowed by the domain decomposition in direction %c\n",
4199                 ddglatnr(dd,dd->cgindex[cg]),dim2char(dim));
4200     }
4201     fprintf(fplog,"distance out of cell %f\n",
4202             dir==1 ? pos_d - comm->cell_x1[dim] : pos_d - comm->cell_x0[dim]);
4203     if (bHaveLimitdAndCMOld)
4204     {
4205         fprintf(fplog,"Old coordinates: %8.3f %8.3f %8.3f\n",
4206                 cm_old[XX],cm_old[YY],cm_old[ZZ]);
4207     }
4208     fprintf(fplog,"New coordinates: %8.3f %8.3f %8.3f\n",
4209             cm_new[XX],cm_new[YY],cm_new[ZZ]);
4210     fprintf(fplog,"Old cell boundaries in direction %c: %8.3f %8.3f\n",
4211             dim2char(dim),
4212             comm->old_cell_x0[dim],comm->old_cell_x1[dim]);
4213     fprintf(fplog,"New cell boundaries in direction %c: %8.3f %8.3f\n",
4214             dim2char(dim),
4215             comm->cell_x0[dim],comm->cell_x1[dim]);
4216 }
4217
4218 static void cg_move_error(FILE *fplog,
4219                           gmx_domdec_t *dd,
4220                           gmx_large_int_t step,int cg,int dim,int dir,
4221                           gmx_bool bHaveLimitdAndCMOld,real limitd,
4222                           rvec cm_old,rvec cm_new,real pos_d)
4223 {
4224     if (fplog)
4225     {
4226         print_cg_move(fplog, dd,step,cg,dim,dir,
4227                       bHaveLimitdAndCMOld,limitd,cm_old,cm_new,pos_d);
4228     }
4229     print_cg_move(stderr,dd,step,cg,dim,dir,
4230                   bHaveLimitdAndCMOld,limitd,cm_old,cm_new,pos_d);
4231     gmx_fatal(FARGS,
4232               "A charge group moved too far between two domain decomposition steps\n"
4233               "This usually means that your system is not well equilibrated");
4234 }
4235
4236 static void rotate_state_atom(t_state *state,int a)
4237 {
4238     int est;
4239
4240     for(est=0; est<estNR; est++)
4241     {
4242         if (EST_DISTR(est) && (state->flags & (1<<est))) {
4243             switch (est) {
4244             case estX:
4245                 /* Rotate the complete state; for a rectangular box only */
4246                 state->x[a][YY] = state->box[YY][YY] - state->x[a][YY];
4247                 state->x[a][ZZ] = state->box[ZZ][ZZ] - state->x[a][ZZ];
4248                 break;
4249             case estV:
4250                 state->v[a][YY] = -state->v[a][YY];
4251                 state->v[a][ZZ] = -state->v[a][ZZ];
4252                 break;
4253             case estSDX:
4254                 state->sd_X[a][YY] = -state->sd_X[a][YY];
4255                 state->sd_X[a][ZZ] = -state->sd_X[a][ZZ];
4256                 break;
4257             case estCGP:
4258                 state->cg_p[a][YY] = -state->cg_p[a][YY];
4259                 state->cg_p[a][ZZ] = -state->cg_p[a][ZZ];
4260                 break;
4261             case estDISRE_INITF:
4262             case estDISRE_RM3TAV:
4263             case estORIRE_INITF:
4264             case estORIRE_DTAV:
4265                 /* These are distances, so not affected by rotation */
4266                 break;
4267             default:
4268                 gmx_incons("Unknown state entry encountered in rotate_state_atom");
4269             }
4270         }
4271     }
4272 }
4273
4274 static int *get_moved(gmx_domdec_comm_t *comm,int natoms)
4275 {
4276     if (natoms > comm->moved_nalloc)
4277     {
4278         /* Contents should be preserved here */
4279         comm->moved_nalloc = over_alloc_dd(natoms);
4280         srenew(comm->moved,comm->moved_nalloc);
4281     }
4282
4283     return comm->moved;
4284 }
4285
4286 static void calc_cg_move(FILE *fplog,gmx_large_int_t step,
4287                          gmx_domdec_t *dd,
4288                          t_state *state,
4289                          ivec tric_dir,matrix tcm,
4290                          rvec cell_x0,rvec cell_x1,
4291                          rvec limitd,rvec limit0,rvec limit1,
4292                          const int *cgindex,
4293                          int cg_start,int cg_end,
4294                          rvec *cg_cm,
4295                          int *move)
4296 {
4297     int  npbcdim;
4298     int  c,i,cg,k,k0,k1,d,dim,dim2,dir,d2,d3,d4,cell_d;
4299     int  mc,cdd,nrcg,ncg_recv,nat_recv,nvs,nvr,nvec,vec;
4300     int  flag;
4301     gmx_bool bScrew;
4302     ivec dev;
4303     real inv_ncg,pos_d;
4304     rvec cm_new;
4305
4306     npbcdim = dd->npbcdim;
4307
4308     for(cg=cg_start; cg<cg_end; cg++)
4309     {
4310         k0   = cgindex[cg];
4311         k1   = cgindex[cg+1];
4312         nrcg = k1 - k0;
4313         if (nrcg == 1)
4314         {
4315             copy_rvec(state->x[k0],cm_new);
4316         }
4317         else
4318         {
4319             inv_ncg = 1.0/nrcg;
4320
4321             clear_rvec(cm_new);
4322             for(k=k0; (k<k1); k++)
4323             {
4324                 rvec_inc(cm_new,state->x[k]);
4325             }
4326             for(d=0; (d<DIM); d++)
4327             {
4328                 cm_new[d] = inv_ncg*cm_new[d];
4329             }
4330         }
4331
4332         clear_ivec(dev);
4333         /* Do pbc and check DD cell boundary crossings */
4334         for(d=DIM-1; d>=0; d--)
4335         {
4336             if (dd->nc[d] > 1)
4337             {
4338                 bScrew = (dd->bScrewPBC && d == XX);
4339                 /* Determine the location of this cg in lattice coordinates */
4340                 pos_d = cm_new[d];
4341                 if (tric_dir[d])
4342                 {
4343                     for(d2=d+1; d2<DIM; d2++)
4344                     {
4345                         pos_d += cm_new[d2]*tcm[d2][d];
4346                     }
4347                 }
4348                 /* Put the charge group in the triclinic unit-cell */
4349                 if (pos_d >= cell_x1[d])
4350                 {
4351                     if (pos_d >= limit1[d])
4352                     {
4353                         cg_move_error(fplog,dd,step,cg,d,1,TRUE,limitd[d],
4354                                       cg_cm[cg],cm_new,pos_d);
4355                     }
4356                     dev[d] = 1;
4357                     if (dd->ci[d] == dd->nc[d] - 1)
4358                     {
4359                         rvec_dec(cm_new,state->box[d]);
4360                         if (bScrew)
4361                         {
4362                             cm_new[YY] = state->box[YY][YY] - cm_new[YY];
4363                             cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
4364                         }
4365                         for(k=k0; (k<k1); k++)
4366                         {
4367                             rvec_dec(state->x[k],state->box[d]);
4368                             if (bScrew)
4369                             {
4370                                 rotate_state_atom(state,k);
4371                             }
4372                         }
4373                     }
4374                 }
4375                 else if (pos_d < cell_x0[d])
4376                 {
4377                     if (pos_d < limit0[d])
4378                     {
4379                         cg_move_error(fplog,dd,step,cg,d,-1,TRUE,limitd[d],
4380                                       cg_cm[cg],cm_new,pos_d);
4381                     }
4382                     dev[d] = -1;
4383                     if (dd->ci[d] == 0)
4384                     {
4385                         rvec_inc(cm_new,state->box[d]);
4386                         if (bScrew)
4387                         {
4388                             cm_new[YY] = state->box[YY][YY] - cm_new[YY];
4389                             cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
4390                         }
4391                         for(k=k0; (k<k1); k++)
4392                         {
4393                             rvec_inc(state->x[k],state->box[d]);
4394                             if (bScrew)
4395                             {
4396                                 rotate_state_atom(state,k);
4397                             }
4398                         }
4399                     }
4400                 }
4401             }
4402             else if (d < npbcdim)
4403             {
4404                 /* Put the charge group in the rectangular unit-cell */
4405                 while (cm_new[d] >= state->box[d][d])
4406                 {
4407                     rvec_dec(cm_new,state->box[d]);
4408                     for(k=k0; (k<k1); k++)
4409                     {
4410                         rvec_dec(state->x[k],state->box[d]);
4411                     }
4412                 }
4413                 while (cm_new[d] < 0)
4414                 {
4415                     rvec_inc(cm_new,state->box[d]);
4416                     for(k=k0; (k<k1); k++)
4417                     {
4418                         rvec_inc(state->x[k],state->box[d]);
4419                     }
4420                 }
4421             }
4422         }
4423
4424         copy_rvec(cm_new,cg_cm[cg]);
4425
4426         /* Determine where this cg should go */
4427         flag = 0;
4428         mc = -1;
4429         for(d=0; d<dd->ndim; d++)
4430         {
4431             dim = dd->dim[d];
4432             if (dev[dim] == 1)
4433             {
4434                 flag |= DD_FLAG_FW(d);
4435                 if (mc == -1)
4436                 {
4437                     mc = d*2;
4438                 }
4439             }
4440             else if (dev[dim] == -1)
4441             {
4442                 flag |= DD_FLAG_BW(d);
4443                 if (mc == -1) {
4444                     if (dd->nc[dim] > 2)
4445                     {
4446                         mc = d*2 + 1;
4447                     }
4448                     else
4449                     {
4450                         mc = d*2;
4451                     }
4452                 }
4453             }
4454         }
4455         /* Temporarily store the flag in move */
4456         move[cg] = mc + flag;
4457     }
4458 }
4459
4460 static void dd_redistribute_cg(FILE *fplog,gmx_large_int_t step,
4461                                gmx_domdec_t *dd,ivec tric_dir,
4462                                t_state *state,rvec **f,
4463                                t_forcerec *fr,t_mdatoms *md,
4464                                gmx_bool bCompact,
4465                                t_nrnb *nrnb,
4466                                int *ncg_stay_home,
4467                                int *ncg_moved)
4468 {
4469     int  *move;
4470     int  npbcdim;
4471     int  ncg[DIM*2],nat[DIM*2];
4472     int  c,i,cg,k,k0,k1,d,dim,dim2,dir,d2,d3,d4,cell_d;
4473     int  mc,cdd,nrcg,ncg_recv,nat_recv,nvs,nvr,nvec,vec;
4474     int  sbuf[2],rbuf[2];
4475     int  home_pos_cg,home_pos_at,buf_pos;
4476     int  flag;
4477     gmx_bool bV=FALSE,bSDX=FALSE,bCGP=FALSE;
4478     gmx_bool bScrew;
4479     ivec dev;
4480     real inv_ncg,pos_d;
4481     matrix tcm;
4482     rvec *cg_cm=NULL,cell_x0,cell_x1,limitd,limit0,limit1,cm_new;
4483     atom_id *cgindex;
4484     cginfo_mb_t *cginfo_mb;
4485     gmx_domdec_comm_t *comm;
4486     int  *moved;
4487     int  nthread,thread;
4488
4489     if (dd->bScrewPBC)
4490     {
4491         check_screw_box(state->box);
4492     }
4493
4494     comm  = dd->comm;
4495     if (fr->cutoff_scheme == ecutsGROUP)
4496     {
4497         cg_cm = fr->cg_cm;
4498     }
4499
4500     for(i=0; i<estNR; i++)
4501     {
4502         if (EST_DISTR(i))
4503         {
4504             switch (i)
4505             {
4506             case estX:   /* Always present */            break;
4507             case estV:   bV   = (state->flags & (1<<i)); break;
4508             case estSDX: bSDX = (state->flags & (1<<i)); break;
4509             case estCGP: bCGP = (state->flags & (1<<i)); break;
4510             case estLD_RNG:
4511             case estLD_RNGI:
4512             case estDISRE_INITF:
4513             case estDISRE_RM3TAV:
4514             case estORIRE_INITF:
4515             case estORIRE_DTAV:
4516                 /* No processing required */
4517                 break;
4518             default:
4519             gmx_incons("Unknown state entry encountered in dd_redistribute_cg");
4520             }
4521         }
4522     }
4523
4524     if (dd->ncg_tot > comm->nalloc_int)
4525     {
4526         comm->nalloc_int = over_alloc_dd(dd->ncg_tot);
4527         srenew(comm->buf_int,comm->nalloc_int);
4528     }
4529     move = comm->buf_int;
4530
4531     /* Clear the count */
4532     for(c=0; c<dd->ndim*2; c++)
4533     {
4534         ncg[c] = 0;
4535         nat[c] = 0;
4536     }
4537
4538     npbcdim = dd->npbcdim;
4539
4540     for(d=0; (d<DIM); d++)
4541     {
4542         limitd[d] = dd->comm->cellsize_min[d];
4543         if (d >= npbcdim && dd->ci[d] == 0)
4544         {
4545             cell_x0[d] = -GMX_FLOAT_MAX;
4546         }
4547         else
4548         {
4549             cell_x0[d] = comm->cell_x0[d];
4550         }
4551         if (d >= npbcdim && dd->ci[d] == dd->nc[d] - 1)
4552         {
4553             cell_x1[d] = GMX_FLOAT_MAX;
4554         }
4555         else
4556         {
4557             cell_x1[d] = comm->cell_x1[d];
4558         }
4559         if (d < npbcdim)
4560         {
4561             limit0[d] = comm->old_cell_x0[d] - limitd[d];
4562             limit1[d] = comm->old_cell_x1[d] + limitd[d];
4563         }
4564         else
4565         {
4566             /* We check after communication if a charge group moved
4567              * more than one cell. Set the pre-comm check limit to float_max.
4568              */
4569             limit0[d] = -GMX_FLOAT_MAX;
4570             limit1[d] =  GMX_FLOAT_MAX;
4571         }
4572     }
4573
4574     make_tric_corr_matrix(npbcdim,state->box,tcm);
4575
4576     cgindex = dd->cgindex;
4577
4578     nthread = gmx_omp_nthreads_get(emntDomdec);
4579
4580     /* Compute the center of geometry for all home charge groups
4581      * and put them in the box and determine where they should go.
4582      */
4583 #pragma omp parallel for num_threads(nthread) schedule(static)
4584     for(thread=0; thread<nthread; thread++)
4585     {
4586         calc_cg_move(fplog,step,dd,state,tric_dir,tcm,
4587                      cell_x0,cell_x1,limitd,limit0,limit1,
4588                      cgindex,
4589                      ( thread   *dd->ncg_home)/nthread,
4590                      ((thread+1)*dd->ncg_home)/nthread,
4591                      fr->cutoff_scheme==ecutsGROUP ? cg_cm : state->x,
4592                      move);
4593     }
4594
4595     for(cg=0; cg<dd->ncg_home; cg++)
4596     {
4597         if (move[cg] >= 0)
4598         {
4599             mc = move[cg];
4600             flag     = mc & ~DD_FLAG_NRCG;
4601             mc       = mc & DD_FLAG_NRCG;
4602             move[cg] = mc;
4603
4604             if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
4605             {
4606                 comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
4607                 srenew(comm->cggl_flag[mc],comm->cggl_flag_nalloc[mc]*DD_CGIBS);
4608             }
4609             comm->cggl_flag[mc][ncg[mc]*DD_CGIBS  ] = dd->index_gl[cg];
4610             /* We store the cg size in the lower 16 bits
4611              * and the place where the charge group should go
4612              * in the next 6 bits. This saves some communication volume.
4613              */
4614             nrcg = cgindex[cg+1] - cgindex[cg];
4615             comm->cggl_flag[mc][ncg[mc]*DD_CGIBS+1] = nrcg | flag;
4616             ncg[mc] += 1;
4617             nat[mc] += nrcg;
4618         }
4619     }
4620
4621     inc_nrnb(nrnb,eNR_CGCM,dd->nat_home);
4622     inc_nrnb(nrnb,eNR_RESETX,dd->ncg_home);
4623
4624     *ncg_moved = 0;
4625     for(i=0; i<dd->ndim*2; i++)
4626     {
4627         *ncg_moved += ncg[i];
4628     }
4629
4630     nvec = 1;
4631     if (bV)
4632     {
4633         nvec++;
4634     }
4635     if (bSDX)
4636     {
4637         nvec++;
4638     }
4639     if (bCGP)
4640     {
4641         nvec++;
4642     }
4643
4644     /* Make sure the communication buffers are large enough */
4645     for(mc=0; mc<dd->ndim*2; mc++)
4646     {
4647         nvr = ncg[mc] + nat[mc]*nvec;
4648         if (nvr > comm->cgcm_state_nalloc[mc])
4649         {
4650             comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr);
4651             srenew(comm->cgcm_state[mc],comm->cgcm_state_nalloc[mc]);
4652         }
4653     }
4654
4655     switch (fr->cutoff_scheme)
4656     {
4657     case ecutsGROUP:
4658         /* Recalculating cg_cm might be cheaper than communicating,
4659          * but that could give rise to rounding issues.
4660          */
4661         home_pos_cg =
4662             compact_and_copy_vec_cg(dd->ncg_home,move,cgindex,
4663                                     nvec,cg_cm,comm,bCompact);
4664     break;
4665     case ecutsVERLET:
4666         /* Without charge groups we send the moved atom coordinates
4667          * over twice. This is so the code below can be used without
4668          * many conditionals for both for with and without charge groups.
4669          */
4670         home_pos_cg =
4671             compact_and_copy_vec_cg(dd->ncg_home,move,cgindex,
4672                                     nvec,state->x,comm,FALSE);
4673         if (bCompact)
4674         {
4675             home_pos_cg -= *ncg_moved;
4676         }
4677         break;
4678     default:
4679         gmx_incons("unimplemented");
4680         home_pos_cg = 0;
4681     }
4682
4683     vec = 0;
4684     home_pos_at =
4685         compact_and_copy_vec_at(dd->ncg_home,move,cgindex,
4686                                 nvec,vec++,state->x,comm,bCompact);
4687     if (bV)
4688     {
4689         compact_and_copy_vec_at(dd->ncg_home,move,cgindex,
4690                                 nvec,vec++,state->v,comm,bCompact);
4691     }
4692     if (bSDX)
4693     {
4694         compact_and_copy_vec_at(dd->ncg_home,move,cgindex,
4695                                 nvec,vec++,state->sd_X,comm,bCompact);
4696     }
4697     if (bCGP)
4698     {
4699         compact_and_copy_vec_at(dd->ncg_home,move,cgindex,
4700                                 nvec,vec++,state->cg_p,comm,bCompact);
4701     }
4702
4703     if (bCompact)
4704     {
4705         compact_ind(dd->ncg_home,move,
4706                     dd->index_gl,dd->cgindex,dd->gatindex,
4707                     dd->ga2la,comm->bLocalCG,
4708                     fr->cginfo);
4709     }
4710     else
4711     {
4712         if (fr->cutoff_scheme == ecutsVERLET)
4713         {
4714             moved = get_moved(comm,dd->ncg_home);
4715
4716             for(k=0; k<dd->ncg_home; k++)
4717             {
4718                 moved[k] = 0;
4719             }
4720         }
4721         else
4722         {
4723             moved = fr->ns.grid->cell_index;
4724         }
4725
4726         clear_and_mark_ind(dd->ncg_home,move,
4727                            dd->index_gl,dd->cgindex,dd->gatindex,
4728                            dd->ga2la,comm->bLocalCG,
4729                            moved);
4730     }
4731
4732     cginfo_mb = fr->cginfo_mb;
4733
4734     *ncg_stay_home = home_pos_cg;
4735     for(d=0; d<dd->ndim; d++)
4736     {
4737         dim = dd->dim[d];
4738         ncg_recv = 0;
4739         nat_recv = 0;
4740         nvr      = 0;
4741         for(dir=0; dir<(dd->nc[dim]==2 ? 1 : 2); dir++)
4742         {
4743             cdd = d*2 + dir;
4744             /* Communicate the cg and atom counts */
4745             sbuf[0] = ncg[cdd];
4746             sbuf[1] = nat[cdd];
4747             if (debug)
4748             {
4749                 fprintf(debug,"Sending ddim %d dir %d: ncg %d nat %d\n",
4750                         d,dir,sbuf[0],sbuf[1]);
4751             }
4752             dd_sendrecv_int(dd, d, dir, sbuf, 2, rbuf, 2);
4753
4754             if ((ncg_recv+rbuf[0])*DD_CGIBS > comm->nalloc_int)
4755             {
4756                 comm->nalloc_int = over_alloc_dd((ncg_recv+rbuf[0])*DD_CGIBS);
4757                 srenew(comm->buf_int,comm->nalloc_int);
4758             }
4759
4760             /* Communicate the charge group indices, sizes and flags */
4761             dd_sendrecv_int(dd, d, dir,
4762                             comm->cggl_flag[cdd], sbuf[0]*DD_CGIBS,
4763                             comm->buf_int+ncg_recv*DD_CGIBS, rbuf[0]*DD_CGIBS);
4764
4765             nvs = ncg[cdd] + nat[cdd]*nvec;
4766             i   = rbuf[0]  + rbuf[1] *nvec;
4767             vec_rvec_check_alloc(&comm->vbuf,nvr+i);
4768
4769             /* Communicate cgcm and state */
4770             dd_sendrecv_rvec(dd, d, dir,
4771                              comm->cgcm_state[cdd], nvs,
4772                              comm->vbuf.v+nvr, i);
4773             ncg_recv += rbuf[0];
4774             nat_recv += rbuf[1];
4775             nvr      += i;
4776         }
4777
4778         /* Process the received charge groups */
4779         buf_pos = 0;
4780         for(cg=0; cg<ncg_recv; cg++)
4781         {
4782             flag = comm->buf_int[cg*DD_CGIBS+1];
4783
4784             if (dim >= npbcdim && dd->nc[dim] > 2)
4785             {
4786                 /* No pbc in this dim and more than one domain boundary.
4787                  * We do a separate check if a charge group didn't move too far.
4788                  */
4789                 if (((flag & DD_FLAG_FW(d)) &&
4790                      comm->vbuf.v[buf_pos][dim] > cell_x1[dim]) ||
4791                     ((flag & DD_FLAG_BW(d)) &&
4792                      comm->vbuf.v[buf_pos][dim] < cell_x0[dim]))
4793                 {
4794                     cg_move_error(fplog,dd,step,cg,dim,
4795                                   (flag & DD_FLAG_FW(d)) ? 1 : 0,
4796                                    FALSE,0,
4797                                    comm->vbuf.v[buf_pos],
4798                                    comm->vbuf.v[buf_pos],
4799                                    comm->vbuf.v[buf_pos][dim]);
4800                 }
4801             }
4802
4803             mc = -1;
4804             if (d < dd->ndim-1)
4805             {
4806                 /* Check which direction this cg should go */
4807                 for(d2=d+1; (d2<dd->ndim && mc==-1); d2++)
4808                 {
4809                     if (dd->bGridJump)
4810                     {
4811                         /* The cell boundaries for dimension d2 are not equal
4812                          * for each cell row of the lower dimension(s),
4813                          * therefore we might need to redetermine where
4814                          * this cg should go.
4815                          */
4816                         dim2 = dd->dim[d2];
4817                         /* If this cg crosses the box boundary in dimension d2
4818                          * we can use the communicated flag, so we do not
4819                          * have to worry about pbc.
4820                          */
4821                         if (!((dd->ci[dim2] == dd->nc[dim2]-1 &&
4822                                (flag & DD_FLAG_FW(d2))) ||
4823                               (dd->ci[dim2] == 0 &&
4824                                (flag & DD_FLAG_BW(d2)))))
4825                         {
4826                             /* Clear the two flags for this dimension */
4827                             flag &= ~(DD_FLAG_FW(d2) | DD_FLAG_BW(d2));
4828                             /* Determine the location of this cg
4829                              * in lattice coordinates
4830                              */
4831                             pos_d = comm->vbuf.v[buf_pos][dim2];
4832                             if (tric_dir[dim2])
4833                             {
4834                                 for(d3=dim2+1; d3<DIM; d3++)
4835                                 {
4836                                     pos_d +=
4837                                         comm->vbuf.v[buf_pos][d3]*tcm[d3][dim2];
4838                                 }
4839                             }
4840                             /* Check of we are not at the box edge.
4841                              * pbc is only handled in the first step above,
4842                              * but this check could move over pbc while
4843                              * the first step did not due to different rounding.
4844                              */
4845                             if (pos_d >= cell_x1[dim2] &&
4846                                 dd->ci[dim2] != dd->nc[dim2]-1)
4847                             {
4848                                 flag |= DD_FLAG_FW(d2);
4849                             }
4850                             else if (pos_d < cell_x0[dim2] &&
4851                                      dd->ci[dim2] != 0)
4852                             {
4853                                 flag |= DD_FLAG_BW(d2);
4854                             }
4855                             comm->buf_int[cg*DD_CGIBS+1] = flag;
4856                         }
4857                     }
4858                     /* Set to which neighboring cell this cg should go */
4859                     if (flag & DD_FLAG_FW(d2))
4860                     {
4861                         mc = d2*2;
4862                     }
4863                     else if (flag & DD_FLAG_BW(d2))
4864                     {
4865                         if (dd->nc[dd->dim[d2]] > 2)
4866                         {
4867                             mc = d2*2+1;
4868                         }
4869                         else
4870                         {
4871                             mc = d2*2;
4872                         }
4873                     }
4874                 }
4875             }
4876
4877             nrcg = flag & DD_FLAG_NRCG;
4878             if (mc == -1)
4879             {
4880                 if (home_pos_cg+1 > dd->cg_nalloc)
4881                 {
4882                     dd->cg_nalloc = over_alloc_dd(home_pos_cg+1);
4883                     srenew(dd->index_gl,dd->cg_nalloc);
4884                     srenew(dd->cgindex,dd->cg_nalloc+1);
4885                 }
4886                 /* Set the global charge group index and size */
4887                 dd->index_gl[home_pos_cg] = comm->buf_int[cg*DD_CGIBS];
4888                 dd->cgindex[home_pos_cg+1] = dd->cgindex[home_pos_cg] + nrcg;
4889                 /* Copy the state from the buffer */
4890                 dd_check_alloc_ncg(fr,state,f,home_pos_cg+1);
4891                 if (fr->cutoff_scheme == ecutsGROUP)
4892                 {
4893                     cg_cm = fr->cg_cm;
4894                     copy_rvec(comm->vbuf.v[buf_pos],cg_cm[home_pos_cg]);
4895                 }
4896                 buf_pos++;
4897
4898                 /* Set the cginfo */
4899                 fr->cginfo[home_pos_cg] = ddcginfo(cginfo_mb,
4900                                                    dd->index_gl[home_pos_cg]);
4901                 if (comm->bLocalCG)
4902                 {
4903                     comm->bLocalCG[dd->index_gl[home_pos_cg]] = TRUE;
4904                 }
4905
4906                 if (home_pos_at+nrcg > state->nalloc)
4907                 {
4908                     dd_realloc_state(state,f,home_pos_at+nrcg);
4909                 }
4910                 for(i=0; i<nrcg; i++)
4911                 {
4912                     copy_rvec(comm->vbuf.v[buf_pos++],
4913                               state->x[home_pos_at+i]);
4914                 }
4915                 if (bV)
4916                 {
4917                     for(i=0; i<nrcg; i++)
4918                     {
4919                         copy_rvec(comm->vbuf.v[buf_pos++],
4920                                   state->v[home_pos_at+i]);
4921                     }
4922                 }
4923                 if (bSDX)
4924                 {
4925                     for(i=0; i<nrcg; i++)
4926                     {
4927                         copy_rvec(comm->vbuf.v[buf_pos++],
4928                                   state->sd_X[home_pos_at+i]);
4929                     }
4930                 }
4931                 if (bCGP)
4932                 {
4933                     for(i=0; i<nrcg; i++)
4934                     {
4935                         copy_rvec(comm->vbuf.v[buf_pos++],
4936                                   state->cg_p[home_pos_at+i]);
4937                     }
4938                 }
4939                 home_pos_cg += 1;
4940                 home_pos_at += nrcg;
4941             }
4942             else
4943             {
4944                 /* Reallocate the buffers if necessary  */
4945                 if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
4946                 {
4947                     comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
4948                     srenew(comm->cggl_flag[mc],comm->cggl_flag_nalloc[mc]*DD_CGIBS);
4949                 }
4950                 nvr = ncg[mc] + nat[mc]*nvec;
4951                 if (nvr + 1 + nrcg*nvec > comm->cgcm_state_nalloc[mc])
4952                 {
4953                     comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr + 1 + nrcg*nvec);
4954                     srenew(comm->cgcm_state[mc],comm->cgcm_state_nalloc[mc]);
4955                 }
4956                 /* Copy from the receive to the send buffers */
4957                 memcpy(comm->cggl_flag[mc] + ncg[mc]*DD_CGIBS,
4958                        comm->buf_int + cg*DD_CGIBS,
4959                        DD_CGIBS*sizeof(int));
4960                 memcpy(comm->cgcm_state[mc][nvr],
4961                        comm->vbuf.v[buf_pos],
4962                        (1+nrcg*nvec)*sizeof(rvec));
4963                 buf_pos += 1 + nrcg*nvec;
4964                 ncg[mc] += 1;
4965                 nat[mc] += nrcg;
4966             }
4967         }
4968     }
4969
4970     /* With sorting (!bCompact) the indices are now only partially up to date
4971      * and ncg_home and nat_home are not the real count, since there are
4972      * "holes" in the arrays for the charge groups that moved to neighbors.
4973      */
4974     if (fr->cutoff_scheme == ecutsVERLET)
4975     {
4976         moved = get_moved(comm,home_pos_cg);
4977
4978         for(i=dd->ncg_home; i<home_pos_cg; i++)
4979         {
4980             moved[i] = 0;
4981         }
4982     }
4983     dd->ncg_home = home_pos_cg;
4984     dd->nat_home = home_pos_at;
4985
4986     if (debug)
4987     {
4988         fprintf(debug,
4989                 "Finished repartitioning: cgs moved out %d, new home %d\n",
4990                 *ncg_moved,dd->ncg_home-*ncg_moved);
4991
4992     }
4993 }
4994
4995 void dd_cycles_add(gmx_domdec_t *dd,float cycles,int ddCycl)
4996 {
4997     dd->comm->cycl[ddCycl] += cycles;
4998     dd->comm->cycl_n[ddCycl]++;
4999     if (cycles > dd->comm->cycl_max[ddCycl])
5000     {
5001         dd->comm->cycl_max[ddCycl] = cycles;
5002     }
5003 }
5004
5005 static double force_flop_count(t_nrnb *nrnb)
5006 {
5007     int i;
5008     double sum;
5009     const char *name;
5010
5011     sum = 0;
5012     for(i=eNR_NBKERNEL010; i<eNR_NBKERNEL_FREE_ENERGY; i++)
5013     {
5014         /* To get closer to the real timings, we half the count
5015          * for the normal loops and again half it for water loops.
5016          */
5017         name = nrnb_str(i);
5018         if (strstr(name,"W3") != NULL || strstr(name,"W4") != NULL)
5019         {
5020             sum += nrnb->n[i]*0.25*cost_nrnb(i);
5021         }
5022         else
5023         {
5024             sum += nrnb->n[i]*0.50*cost_nrnb(i);
5025         }
5026     }
5027     for(i=eNR_NBKERNEL_FREE_ENERGY; i<=eNR_NB14; i++)
5028     {
5029         name = nrnb_str(i);
5030         if (strstr(name,"W3") != NULL || strstr(name,"W4") != NULL)
5031         sum += nrnb->n[i]*cost_nrnb(i);
5032     }
5033     for(i=eNR_BONDS; i<=eNR_WALLS; i++)
5034     {
5035         sum += nrnb->n[i]*cost_nrnb(i);
5036     }
5037
5038     return sum;
5039 }
5040
5041 void dd_force_flop_start(gmx_domdec_t *dd,t_nrnb *nrnb)
5042 {
5043     if (dd->comm->eFlop)
5044     {
5045         dd->comm->flop -= force_flop_count(nrnb);
5046     }
5047 }
5048 void dd_force_flop_stop(gmx_domdec_t *dd,t_nrnb *nrnb)
5049 {
5050     if (dd->comm->eFlop)
5051     {
5052         dd->comm->flop += force_flop_count(nrnb);
5053         dd->comm->flop_n++;
5054     }
5055 }
5056
5057 static void clear_dd_cycle_counts(gmx_domdec_t *dd)
5058 {
5059     int i;
5060
5061     for(i=0; i<ddCyclNr; i++)
5062     {
5063         dd->comm->cycl[i] = 0;
5064         dd->comm->cycl_n[i] = 0;
5065         dd->comm->cycl_max[i] = 0;
5066     }
5067     dd->comm->flop = 0;
5068     dd->comm->flop_n = 0;
5069 }
5070
5071 static void get_load_distribution(gmx_domdec_t *dd,gmx_wallcycle_t wcycle)
5072 {
5073     gmx_domdec_comm_t *comm;
5074     gmx_domdec_load_t *load;
5075     gmx_domdec_root_t *root=NULL;
5076     int  d,dim,cid,i,pos;
5077     float cell_frac=0,sbuf[DD_NLOAD_MAX];
5078     gmx_bool bSepPME;
5079
5080     if (debug)
5081     {
5082         fprintf(debug,"get_load_distribution start\n");
5083     }
5084
5085     wallcycle_start(wcycle,ewcDDCOMMLOAD);
5086
5087     comm = dd->comm;
5088
5089     bSepPME = (dd->pme_nodeid >= 0);
5090
5091     for(d=dd->ndim-1; d>=0; d--)
5092     {
5093         dim = dd->dim[d];
5094         /* Check if we participate in the communication in this dimension */
5095         if (d == dd->ndim-1 ||
5096             (dd->ci[dd->dim[d+1]]==0 && dd->ci[dd->dim[dd->ndim-1]]==0))
5097         {
5098             load = &comm->load[d];
5099             if (dd->bGridJump)
5100             {
5101                 cell_frac = comm->cell_f1[d] - comm->cell_f0[d];
5102             }
5103             pos = 0;
5104             if (d == dd->ndim-1)
5105             {
5106                 sbuf[pos++] = dd_force_load(comm);
5107                 sbuf[pos++] = sbuf[0];
5108                 if (dd->bGridJump)
5109                 {
5110                     sbuf[pos++] = sbuf[0];
5111                     sbuf[pos++] = cell_frac;
5112                     if (d > 0)
5113                     {
5114                         sbuf[pos++] = comm->cell_f_max0[d];
5115                         sbuf[pos++] = comm->cell_f_min1[d];
5116                     }
5117                 }
5118                 if (bSepPME)
5119                 {
5120                     sbuf[pos++] = comm->cycl[ddCyclPPduringPME];
5121                     sbuf[pos++] = comm->cycl[ddCyclPME];
5122                 }
5123             }
5124             else
5125             {
5126                 sbuf[pos++] = comm->load[d+1].sum;
5127                 sbuf[pos++] = comm->load[d+1].max;
5128                 if (dd->bGridJump)
5129                 {
5130                     sbuf[pos++] = comm->load[d+1].sum_m;
5131                     sbuf[pos++] = comm->load[d+1].cvol_min*cell_frac;
5132                     sbuf[pos++] = comm->load[d+1].flags;
5133                     if (d > 0)
5134                     {
5135                         sbuf[pos++] = comm->cell_f_max0[d];
5136                         sbuf[pos++] = comm->cell_f_min1[d];
5137                     }
5138                 }
5139                 if (bSepPME)
5140                 {
5141                     sbuf[pos++] = comm->load[d+1].mdf;
5142                     sbuf[pos++] = comm->load[d+1].pme;
5143                 }
5144             }
5145             load->nload = pos;
5146             /* Communicate a row in DD direction d.
5147              * The communicators are setup such that the root always has rank 0.
5148              */
5149 #ifdef GMX_MPI
5150             MPI_Gather(sbuf      ,load->nload*sizeof(float),MPI_BYTE,
5151                        load->load,load->nload*sizeof(float),MPI_BYTE,
5152                        0,comm->mpi_comm_load[d]);
5153 #endif
5154             if (dd->ci[dim] == dd->master_ci[dim])
5155             {
5156                 /* We are the root, process this row */
5157                 if (comm->bDynLoadBal)
5158                 {
5159                     root = comm->root[d];
5160                 }
5161                 load->sum = 0;
5162                 load->max = 0;
5163                 load->sum_m = 0;
5164                 load->cvol_min = 1;
5165                 load->flags = 0;
5166                 load->mdf = 0;
5167                 load->pme = 0;
5168                 pos = 0;
5169                 for(i=0; i<dd->nc[dim]; i++)
5170                 {
5171                     load->sum += load->load[pos++];
5172                     load->max = max(load->max,load->load[pos]);
5173                     pos++;
5174                     if (dd->bGridJump)
5175                     {
5176                         if (root->bLimited)
5177                         {
5178                             /* This direction could not be load balanced properly,
5179                              * therefore we need to use the maximum iso the average load.
5180                              */
5181                             load->sum_m = max(load->sum_m,load->load[pos]);
5182                         }
5183                         else
5184                         {
5185                             load->sum_m += load->load[pos];
5186                         }
5187                         pos++;
5188                         load->cvol_min = min(load->cvol_min,load->load[pos]);
5189                         pos++;
5190                         if (d < dd->ndim-1)
5191                         {
5192                             load->flags = (int)(load->load[pos++] + 0.5);
5193                         }
5194                         if (d > 0)
5195                         {
5196                             root->cell_f_max0[i] = load->load[pos++];
5197                             root->cell_f_min1[i] = load->load[pos++];
5198                         }
5199                     }
5200                     if (bSepPME)
5201                     {
5202                         load->mdf = max(load->mdf,load->load[pos]);
5203                         pos++;
5204                         load->pme = max(load->pme,load->load[pos]);
5205                         pos++;
5206                     }
5207                 }
5208                 if (comm->bDynLoadBal && root->bLimited)
5209                 {
5210                     load->sum_m *= dd->nc[dim];
5211                     load->flags |= (1<<d);
5212                 }
5213             }
5214         }
5215     }
5216
5217     if (DDMASTER(dd))
5218     {
5219         comm->nload      += dd_load_count(comm);
5220         comm->load_step  += comm->cycl[ddCyclStep];
5221         comm->load_sum   += comm->load[0].sum;
5222         comm->load_max   += comm->load[0].max;
5223         if (comm->bDynLoadBal)
5224         {
5225             for(d=0; d<dd->ndim; d++)
5226             {
5227                 if (comm->load[0].flags & (1<<d))
5228                 {
5229                     comm->load_lim[d]++;
5230                 }
5231             }
5232         }
5233         if (bSepPME)
5234         {
5235             comm->load_mdf += comm->load[0].mdf;
5236             comm->load_pme += comm->load[0].pme;
5237         }
5238     }
5239
5240     wallcycle_stop(wcycle,ewcDDCOMMLOAD);
5241
5242     if (debug)
5243     {
5244         fprintf(debug,"get_load_distribution finished\n");
5245     }
5246 }
5247
5248 static float dd_force_imb_perf_loss(gmx_domdec_t *dd)
5249 {
5250     /* Return the relative performance loss on the total run time
5251      * due to the force calculation load imbalance.
5252      */
5253     if (dd->comm->nload > 0)
5254     {
5255         return
5256             (dd->comm->load_max*dd->nnodes - dd->comm->load_sum)/
5257             (dd->comm->load_step*dd->nnodes);
5258     }
5259     else
5260     {
5261         return 0;
5262     }
5263 }
5264
5265 static void print_dd_load_av(FILE *fplog,gmx_domdec_t *dd)
5266 {
5267     char  buf[STRLEN];
5268     int   npp,npme,nnodes,d,limp;
5269     float imbal,pme_f_ratio,lossf,lossp=0;
5270     gmx_bool  bLim;
5271     gmx_domdec_comm_t *comm;
5272
5273     comm = dd->comm;
5274     if (DDMASTER(dd) && comm->nload > 0)
5275     {
5276         npp    = dd->nnodes;
5277         npme   = (dd->pme_nodeid >= 0) ? comm->npmenodes : 0;
5278         nnodes = npp + npme;
5279         imbal = comm->load_max*npp/comm->load_sum - 1;
5280         lossf = dd_force_imb_perf_loss(dd);
5281         sprintf(buf," Average load imbalance: %.1f %%\n",imbal*100);
5282         fprintf(fplog,"%s",buf);
5283         fprintf(stderr,"\n");
5284         fprintf(stderr,"%s",buf);
5285         sprintf(buf," Part of the total run time spent waiting due to load imbalance: %.1f %%\n",lossf*100);
5286         fprintf(fplog,"%s",buf);
5287         fprintf(stderr,"%s",buf);
5288         bLim = FALSE;
5289         if (comm->bDynLoadBal)
5290         {
5291             sprintf(buf," Steps where the load balancing was limited by -rdd, -rcon and/or -dds:");
5292             for(d=0; d<dd->ndim; d++)
5293             {
5294                 limp = (200*comm->load_lim[d]+1)/(2*comm->nload);
5295                 sprintf(buf+strlen(buf)," %c %d %%",dim2char(dd->dim[d]),limp);
5296                 if (limp >= 50)
5297                 {
5298                     bLim = TRUE;
5299                 }
5300             }
5301             sprintf(buf+strlen(buf),"\n");
5302             fprintf(fplog,"%s",buf);
5303             fprintf(stderr,"%s",buf);
5304         }
5305         if (npme > 0)
5306         {
5307             pme_f_ratio = comm->load_pme/comm->load_mdf;
5308             lossp = (comm->load_pme -comm->load_mdf)/comm->load_step;
5309             if (lossp <= 0)
5310             {
5311                 lossp *= (float)npme/(float)nnodes;
5312             }
5313             else
5314             {
5315                 lossp *= (float)npp/(float)nnodes;
5316             }
5317             sprintf(buf," Average PME mesh/force load: %5.3f\n",pme_f_ratio);
5318             fprintf(fplog,"%s",buf);
5319             fprintf(stderr,"%s",buf);
5320             sprintf(buf," Part of the total run time spent waiting due to PP/PME imbalance: %.1f %%\n",fabs(lossp)*100);
5321             fprintf(fplog,"%s",buf);
5322             fprintf(stderr,"%s",buf);
5323         }
5324         fprintf(fplog,"\n");
5325         fprintf(stderr,"\n");
5326
5327         if (lossf >= DD_PERF_LOSS)
5328         {
5329             sprintf(buf,
5330                     "NOTE: %.1f %% performance was lost due to load imbalance\n"
5331                     "      in the domain decomposition.\n",lossf*100);
5332             if (!comm->bDynLoadBal)
5333             {
5334                 sprintf(buf+strlen(buf),"      You might want to use dynamic load balancing (option -dlb.)\n");
5335             }
5336             else if (bLim)
5337             {
5338                 sprintf(buf+strlen(buf),"      You might want to decrease the cell size limit (options -rdd, -rcon and/or -dds).\n");
5339             }
5340             fprintf(fplog,"%s\n",buf);
5341             fprintf(stderr,"%s\n",buf);
5342         }
5343         if (npme > 0 && fabs(lossp) >= DD_PERF_LOSS)
5344         {
5345             sprintf(buf,
5346                     "NOTE: %.1f %% performance was lost because the PME nodes\n"
5347                     "      had %s work to do than the PP nodes.\n"
5348                     "      You might want to %s the number of PME nodes\n"
5349                     "      or %s the cut-off and the grid spacing.\n",
5350                     fabs(lossp*100),
5351                     (lossp < 0) ? "less"     : "more",
5352                     (lossp < 0) ? "decrease" : "increase",
5353                     (lossp < 0) ? "decrease" : "increase");
5354             fprintf(fplog,"%s\n",buf);
5355             fprintf(stderr,"%s\n",buf);
5356         }
5357     }
5358 }
5359
5360 static float dd_vol_min(gmx_domdec_t *dd)
5361 {
5362     return dd->comm->load[0].cvol_min*dd->nnodes;
5363 }
5364
5365 static gmx_bool dd_load_flags(gmx_domdec_t *dd)
5366 {
5367     return dd->comm->load[0].flags;
5368 }
5369
5370 static float dd_f_imbal(gmx_domdec_t *dd)
5371 {
5372     return dd->comm->load[0].max*dd->nnodes/dd->comm->load[0].sum - 1;
5373 }
5374
5375 float dd_pme_f_ratio(gmx_domdec_t *dd)
5376 {
5377     if (dd->comm->cycl_n[ddCyclPME] > 0)
5378     {
5379         return dd->comm->load[0].pme/dd->comm->load[0].mdf;
5380     }
5381     else
5382     {
5383         return -1.0;
5384     }
5385 }
5386
5387 static void dd_print_load(FILE *fplog,gmx_domdec_t *dd,gmx_large_int_t step)
5388 {
5389     int flags,d;
5390     char buf[22];
5391
5392     flags = dd_load_flags(dd);
5393     if (flags)
5394     {
5395         fprintf(fplog,
5396                 "DD  load balancing is limited by minimum cell size in dimension");
5397         for(d=0; d<dd->ndim; d++)
5398         {
5399             if (flags & (1<<d))
5400             {
5401                 fprintf(fplog," %c",dim2char(dd->dim[d]));
5402             }
5403         }
5404         fprintf(fplog,"\n");
5405     }
5406     fprintf(fplog,"DD  step %s",gmx_step_str(step,buf));
5407     if (dd->comm->bDynLoadBal)
5408     {
5409         fprintf(fplog,"  vol min/aver %5.3f%c",
5410                 dd_vol_min(dd),flags ? '!' : ' ');
5411     }
5412     fprintf(fplog," load imb.: force %4.1f%%",dd_f_imbal(dd)*100);
5413     if (dd->comm->cycl_n[ddCyclPME])
5414     {
5415         fprintf(fplog,"  pme mesh/force %5.3f",dd_pme_f_ratio(dd));
5416     }
5417     fprintf(fplog,"\n\n");
5418 }
5419
5420 static void dd_print_load_verbose(gmx_domdec_t *dd)
5421 {
5422     if (dd->comm->bDynLoadBal)
5423     {
5424         fprintf(stderr,"vol %4.2f%c ",
5425                 dd_vol_min(dd),dd_load_flags(dd) ? '!' : ' ');
5426     }
5427     fprintf(stderr,"imb F %2d%% ",(int)(dd_f_imbal(dd)*100+0.5));
5428     if (dd->comm->cycl_n[ddCyclPME])
5429     {
5430         fprintf(stderr,"pme/F %4.2f ",dd_pme_f_ratio(dd));
5431     }
5432 }
5433
5434 #ifdef GMX_MPI
5435 static void make_load_communicator(gmx_domdec_t *dd, int dim_ind,ivec loc)
5436 {
5437     MPI_Comm  c_row;
5438     int  dim, i, rank;
5439     ivec loc_c;
5440     gmx_domdec_root_t *root;
5441     gmx_bool bPartOfGroup = FALSE;
5442
5443     dim = dd->dim[dim_ind];
5444     copy_ivec(loc,loc_c);
5445     for(i=0; i<dd->nc[dim]; i++)
5446     {
5447         loc_c[dim] = i;
5448         rank = dd_index(dd->nc,loc_c);
5449         if (rank == dd->rank)
5450         {
5451             /* This process is part of the group */
5452             bPartOfGroup = TRUE;
5453         }
5454     }
5455     MPI_Comm_split(dd->mpi_comm_all, bPartOfGroup?0:MPI_UNDEFINED, dd->rank,
5456                    &c_row);
5457     if (bPartOfGroup)
5458     {
5459         dd->comm->mpi_comm_load[dim_ind] = c_row;
5460         if (dd->comm->eDLB != edlbNO)
5461         {
5462             if (dd->ci[dim] == dd->master_ci[dim])
5463             {
5464                 /* This is the root process of this row */
5465                 snew(dd->comm->root[dim_ind],1);
5466                 root = dd->comm->root[dim_ind];
5467                 snew(root->cell_f,DD_CELL_F_SIZE(dd,dim_ind));
5468                 snew(root->old_cell_f,dd->nc[dim]+1);
5469                 snew(root->bCellMin,dd->nc[dim]);
5470                 if (dim_ind > 0)
5471                 {
5472                     snew(root->cell_f_max0,dd->nc[dim]);
5473                     snew(root->cell_f_min1,dd->nc[dim]);
5474                     snew(root->bound_min,dd->nc[dim]);
5475                     snew(root->bound_max,dd->nc[dim]);
5476                 }
5477                 snew(root->buf_ncd,dd->nc[dim]);
5478             }
5479             else
5480             {
5481                 /* This is not a root process, we only need to receive cell_f */
5482                 snew(dd->comm->cell_f_row,DD_CELL_F_SIZE(dd,dim_ind));
5483             }
5484         }
5485         if (dd->ci[dim] == dd->master_ci[dim])
5486         {
5487             snew(dd->comm->load[dim_ind].load,dd->nc[dim]*DD_NLOAD_MAX);
5488         }
5489     }
5490 }
5491 #endif
5492
5493 static void make_load_communicators(gmx_domdec_t *dd)
5494 {
5495 #ifdef GMX_MPI
5496   int  dim0,dim1,i,j;
5497   ivec loc;
5498
5499   if (debug)
5500     fprintf(debug,"Making load communicators\n");
5501
5502   snew(dd->comm->load,dd->ndim);
5503   snew(dd->comm->mpi_comm_load,dd->ndim);
5504
5505   clear_ivec(loc);
5506   make_load_communicator(dd,0,loc);
5507   if (dd->ndim > 1) {
5508     dim0 = dd->dim[0];
5509     for(i=0; i<dd->nc[dim0]; i++) {
5510       loc[dim0] = i;
5511       make_load_communicator(dd,1,loc);
5512     }
5513   }
5514   if (dd->ndim > 2) {
5515     dim0 = dd->dim[0];
5516     for(i=0; i<dd->nc[dim0]; i++) {
5517       loc[dim0] = i;
5518       dim1 = dd->dim[1];
5519       for(j=0; j<dd->nc[dim1]; j++) {
5520           loc[dim1] = j;
5521           make_load_communicator(dd,2,loc);
5522       }
5523     }
5524   }
5525
5526   if (debug)
5527     fprintf(debug,"Finished making load communicators\n");
5528 #endif
5529 }
5530
5531 void setup_dd_grid(FILE *fplog,gmx_domdec_t *dd)
5532 {
5533     gmx_bool bZYX;
5534     int  d,dim,i,j,m;
5535     ivec tmp,s;
5536     int  nzone,nzonep;
5537     ivec dd_zp[DD_MAXIZONE];
5538     gmx_domdec_zones_t *zones;
5539     gmx_domdec_ns_ranges_t *izone;
5540
5541     for(d=0; d<dd->ndim; d++)
5542     {
5543         dim = dd->dim[d];
5544         copy_ivec(dd->ci,tmp);
5545         tmp[dim] = (tmp[dim] + 1) % dd->nc[dim];
5546         dd->neighbor[d][0] = ddcoord2ddnodeid(dd,tmp);
5547         copy_ivec(dd->ci,tmp);
5548         tmp[dim] = (tmp[dim] - 1 + dd->nc[dim]) % dd->nc[dim];
5549         dd->neighbor[d][1] = ddcoord2ddnodeid(dd,tmp);
5550         if (debug)
5551         {
5552             fprintf(debug,"DD rank %d neighbor ranks in dir %d are + %d - %d\n",
5553                     dd->rank,dim,
5554                     dd->neighbor[d][0],
5555                     dd->neighbor[d][1]);
5556         }
5557     }
5558
5559     if (DDMASTER(dd))
5560     {
5561         fprintf(stderr,"Making %dD domain decomposition %d x %d x %d\n",
5562             dd->ndim,dd->nc[XX],dd->nc[YY],dd->nc[ZZ]);
5563     }
5564     if (fplog)
5565     {
5566         fprintf(fplog,"\nMaking %dD domain decomposition grid %d x %d x %d, home cell index %d %d %d\n\n",
5567                 dd->ndim,
5568                 dd->nc[XX],dd->nc[YY],dd->nc[ZZ],
5569                 dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
5570     }
5571     switch (dd->ndim)
5572     {
5573     case 3:
5574         nzone  = dd_z3n;
5575         nzonep = dd_zp3n;
5576         for(i=0; i<nzonep; i++)
5577         {
5578             copy_ivec(dd_zp3[i],dd_zp[i]);
5579         }
5580         break;
5581     case 2:
5582         nzone  = dd_z2n;
5583         nzonep = dd_zp2n;
5584         for(i=0; i<nzonep; i++)
5585         {
5586             copy_ivec(dd_zp2[i],dd_zp[i]);
5587         }
5588         break;
5589     case 1:
5590         nzone  = dd_z1n;
5591         nzonep = dd_zp1n;
5592         for(i=0; i<nzonep; i++)
5593         {
5594             copy_ivec(dd_zp1[i],dd_zp[i]);
5595         }
5596         break;
5597     default:
5598         gmx_fatal(FARGS,"Can only do 1, 2 or 3D domain decomposition");
5599         nzone = 0;
5600         nzonep = 0;
5601     }
5602
5603     zones = &dd->comm->zones;
5604
5605     for(i=0; i<nzone; i++)
5606     {
5607         m = 0;
5608         clear_ivec(zones->shift[i]);
5609         for(d=0; d<dd->ndim; d++)
5610         {
5611             zones->shift[i][dd->dim[d]] = dd_zo[i][m++];
5612         }
5613     }
5614
5615     zones->n = nzone;
5616     for(i=0; i<nzone; i++)
5617     {
5618         for(d=0; d<DIM; d++)
5619         {
5620             s[d] = dd->ci[d] - zones->shift[i][d];
5621             if (s[d] < 0)
5622             {
5623                 s[d] += dd->nc[d];
5624             }
5625             else if (s[d] >= dd->nc[d])
5626             {
5627                 s[d] -= dd->nc[d];
5628             }
5629         }
5630     }
5631     zones->nizone = nzonep;
5632     for(i=0; i<zones->nizone; i++)
5633     {
5634         if (dd_zp[i][0] != i)
5635         {
5636             gmx_fatal(FARGS,"Internal inconsistency in the dd grid setup");
5637         }
5638         izone = &zones->izone[i];
5639         izone->j0 = dd_zp[i][1];
5640         izone->j1 = dd_zp[i][2];
5641         for(dim=0; dim<DIM; dim++)
5642         {
5643             if (dd->nc[dim] == 1)
5644             {
5645                 /* All shifts should be allowed */
5646                 izone->shift0[dim] = -1;
5647                 izone->shift1[dim] = 1;
5648             }
5649             else
5650             {
5651                 /*
5652                   izone->shift0[d] = 0;
5653                   izone->shift1[d] = 0;
5654                   for(j=izone->j0; j<izone->j1; j++) {
5655                   if (dd->shift[j][d] > dd->shift[i][d])
5656                   izone->shift0[d] = -1;
5657                   if (dd->shift[j][d] < dd->shift[i][d])
5658                   izone->shift1[d] = 1;
5659                   }
5660                 */
5661
5662                 int shift_diff;
5663
5664                 /* Assume the shift are not more than 1 cell */
5665                 izone->shift0[dim] = 1;
5666                 izone->shift1[dim] = -1;
5667                 for(j=izone->j0; j<izone->j1; j++)
5668                 {
5669                     shift_diff = zones->shift[j][dim] - zones->shift[i][dim];
5670                     if (shift_diff < izone->shift0[dim])
5671                     {
5672                         izone->shift0[dim] = shift_diff;
5673                     }
5674                     if (shift_diff > izone->shift1[dim])
5675                     {
5676                         izone->shift1[dim] = shift_diff;
5677                     }
5678                 }
5679             }
5680         }
5681     }
5682
5683     if (dd->comm->eDLB != edlbNO)
5684     {
5685         snew(dd->comm->root,dd->ndim);
5686     }
5687
5688     if (dd->comm->bRecordLoad)
5689     {
5690         make_load_communicators(dd);
5691     }
5692 }
5693
5694 static void make_pp_communicator(FILE *fplog,t_commrec *cr,int reorder)
5695 {
5696     gmx_domdec_t *dd;
5697     gmx_domdec_comm_t *comm;
5698     int  i,rank,*buf;
5699     ivec periods;
5700 #ifdef GMX_MPI
5701     MPI_Comm comm_cart;
5702 #endif
5703
5704     dd = cr->dd;
5705     comm = dd->comm;
5706
5707 #ifdef GMX_MPI
5708     if (comm->bCartesianPP)
5709     {
5710         /* Set up cartesian communication for the particle-particle part */
5711         if (fplog)
5712         {
5713             fprintf(fplog,"Will use a Cartesian communicator: %d x %d x %d\n",
5714                     dd->nc[XX],dd->nc[YY],dd->nc[ZZ]);
5715         }
5716
5717         for(i=0; i<DIM; i++)
5718         {
5719             periods[i] = TRUE;
5720         }
5721         MPI_Cart_create(cr->mpi_comm_mygroup,DIM,dd->nc,periods,reorder,
5722                         &comm_cart);
5723         /* We overwrite the old communicator with the new cartesian one */
5724         cr->mpi_comm_mygroup = comm_cart;
5725     }
5726
5727     dd->mpi_comm_all = cr->mpi_comm_mygroup;
5728     MPI_Comm_rank(dd->mpi_comm_all,&dd->rank);
5729
5730     if (comm->bCartesianPP_PME)
5731     {
5732         /* Since we want to use the original cartesian setup for sim,
5733          * and not the one after split, we need to make an index.
5734          */
5735         snew(comm->ddindex2ddnodeid,dd->nnodes);
5736         comm->ddindex2ddnodeid[dd_index(dd->nc,dd->ci)] = dd->rank;
5737         gmx_sumi(dd->nnodes,comm->ddindex2ddnodeid,cr);
5738         /* Get the rank of the DD master,
5739          * above we made sure that the master node is a PP node.
5740          */
5741         if (MASTER(cr))
5742         {
5743             rank = dd->rank;
5744         }
5745         else
5746         {
5747             rank = 0;
5748         }
5749         MPI_Allreduce(&rank,&dd->masterrank,1,MPI_INT,MPI_SUM,dd->mpi_comm_all);
5750     }
5751     else if (comm->bCartesianPP)
5752     {
5753         if (cr->npmenodes == 0)
5754         {
5755             /* The PP communicator is also
5756              * the communicator for this simulation
5757              */
5758             cr->mpi_comm_mysim = cr->mpi_comm_mygroup;
5759         }
5760         cr->nodeid = dd->rank;
5761
5762         MPI_Cart_coords(dd->mpi_comm_all,dd->rank,DIM,dd->ci);
5763
5764         /* We need to make an index to go from the coordinates
5765          * to the nodeid of this simulation.
5766          */
5767         snew(comm->ddindex2simnodeid,dd->nnodes);
5768         snew(buf,dd->nnodes);
5769         if (cr->duty & DUTY_PP)
5770         {
5771             buf[dd_index(dd->nc,dd->ci)] = cr->sim_nodeid;
5772         }
5773         /* Communicate the ddindex to simulation nodeid index */
5774         MPI_Allreduce(buf,comm->ddindex2simnodeid,dd->nnodes,MPI_INT,MPI_SUM,
5775                       cr->mpi_comm_mysim);
5776         sfree(buf);
5777
5778         /* Determine the master coordinates and rank.
5779          * The DD master should be the same node as the master of this sim.
5780          */
5781         for(i=0; i<dd->nnodes; i++)
5782         {
5783             if (comm->ddindex2simnodeid[i] == 0)
5784             {
5785                 ddindex2xyz(dd->nc,i,dd->master_ci);
5786                 MPI_Cart_rank(dd->mpi_comm_all,dd->master_ci,&dd->masterrank);
5787             }
5788         }
5789         if (debug)
5790         {
5791             fprintf(debug,"The master rank is %d\n",dd->masterrank);
5792         }
5793     }
5794     else
5795     {
5796         /* No Cartesian communicators */
5797         /* We use the rank in dd->comm->all as DD index */
5798         ddindex2xyz(dd->nc,dd->rank,dd->ci);
5799         /* The simulation master nodeid is 0, so the DD master rank is also 0 */
5800         dd->masterrank = 0;
5801         clear_ivec(dd->master_ci);
5802     }
5803 #endif
5804
5805     if (fplog)
5806     {
5807         fprintf(fplog,
5808                 "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
5809                 dd->rank,dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
5810     }
5811     if (debug)
5812     {
5813         fprintf(debug,
5814                 "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
5815                 dd->rank,dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
5816     }
5817 }
5818
5819 static void receive_ddindex2simnodeid(t_commrec *cr)
5820 {
5821     gmx_domdec_t *dd;
5822
5823     gmx_domdec_comm_t *comm;
5824     int  *buf;
5825
5826     dd = cr->dd;
5827     comm = dd->comm;
5828
5829 #ifdef GMX_MPI
5830     if (!comm->bCartesianPP_PME && comm->bCartesianPP)
5831     {
5832         snew(comm->ddindex2simnodeid,dd->nnodes);
5833         snew(buf,dd->nnodes);
5834         if (cr->duty & DUTY_PP)
5835         {
5836             buf[dd_index(dd->nc,dd->ci)] = cr->sim_nodeid;
5837         }
5838 #ifdef GMX_MPI
5839         /* Communicate the ddindex to simulation nodeid index */
5840         MPI_Allreduce(buf,comm->ddindex2simnodeid,dd->nnodes,MPI_INT,MPI_SUM,
5841                       cr->mpi_comm_mysim);
5842 #endif
5843         sfree(buf);
5844     }
5845 #endif
5846 }
5847
5848 static gmx_domdec_master_t *init_gmx_domdec_master_t(gmx_domdec_t *dd,
5849                                                      int ncg,int natoms)
5850 {
5851     gmx_domdec_master_t *ma;
5852     int i;
5853
5854     snew(ma,1);
5855
5856     snew(ma->ncg,dd->nnodes);
5857     snew(ma->index,dd->nnodes+1);
5858     snew(ma->cg,ncg);
5859     snew(ma->nat,dd->nnodes);
5860     snew(ma->ibuf,dd->nnodes*2);
5861     snew(ma->cell_x,DIM);
5862     for(i=0; i<DIM; i++)
5863     {
5864         snew(ma->cell_x[i],dd->nc[i]+1);
5865     }
5866
5867     if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
5868     {
5869         ma->vbuf = NULL;
5870     }
5871     else
5872     {
5873         snew(ma->vbuf,natoms);
5874     }
5875
5876     return ma;
5877 }
5878
5879 static void split_communicator(FILE *fplog,t_commrec *cr,int dd_node_order,
5880                                int reorder)
5881 {
5882     gmx_domdec_t *dd;
5883     gmx_domdec_comm_t *comm;
5884     int  i,rank;
5885     gmx_bool bDiv[DIM];
5886     ivec periods;
5887 #ifdef GMX_MPI
5888     MPI_Comm comm_cart;
5889 #endif
5890
5891     dd = cr->dd;
5892     comm = dd->comm;
5893
5894     if (comm->bCartesianPP)
5895     {
5896         for(i=1; i<DIM; i++)
5897         {
5898             bDiv[i] = ((cr->npmenodes*dd->nc[i]) % (dd->nnodes) == 0);
5899         }
5900         if (bDiv[YY] || bDiv[ZZ])
5901         {
5902             comm->bCartesianPP_PME = TRUE;
5903             /* If we have 2D PME decomposition, which is always in x+y,
5904              * we stack the PME only nodes in z.
5905              * Otherwise we choose the direction that provides the thinnest slab
5906              * of PME only nodes as this will have the least effect
5907              * on the PP communication.
5908              * But for the PME communication the opposite might be better.
5909              */
5910             if (bDiv[ZZ] && (comm->npmenodes_y > 1 ||
5911                              !bDiv[YY] ||
5912                              dd->nc[YY] > dd->nc[ZZ]))
5913             {
5914                 comm->cartpmedim = ZZ;
5915             }
5916             else
5917             {
5918                 comm->cartpmedim = YY;
5919             }
5920             comm->ntot[comm->cartpmedim]
5921                 += (cr->npmenodes*dd->nc[comm->cartpmedim])/dd->nnodes;
5922         }
5923         else if (fplog)
5924         {
5925             fprintf(fplog,"#pmenodes (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n",cr->npmenodes,dd->nc[XX],dd->nc[YY],dd->nc[XX],dd->nc[ZZ]);
5926             fprintf(fplog,
5927                     "Will not use a Cartesian communicator for PP <-> PME\n\n");
5928         }
5929     }
5930
5931 #ifdef GMX_MPI
5932     if (comm->bCartesianPP_PME)
5933     {
5934         if (fplog)
5935         {
5936             fprintf(fplog,"Will use a Cartesian communicator for PP <-> PME: %d x %d x %d\n",comm->ntot[XX],comm->ntot[YY],comm->ntot[ZZ]);
5937         }
5938
5939         for(i=0; i<DIM; i++)
5940         {
5941             periods[i] = TRUE;
5942         }
5943         MPI_Cart_create(cr->mpi_comm_mysim,DIM,comm->ntot,periods,reorder,
5944                         &comm_cart);
5945
5946         MPI_Comm_rank(comm_cart,&rank);
5947         if (MASTERNODE(cr) && rank != 0)
5948         {
5949             gmx_fatal(FARGS,"MPI rank 0 was renumbered by MPI_Cart_create, we do not allow this");
5950         }
5951
5952         /* With this assigment we loose the link to the original communicator
5953          * which will usually be MPI_COMM_WORLD, unless have multisim.
5954          */
5955         cr->mpi_comm_mysim = comm_cart;
5956         cr->sim_nodeid = rank;
5957
5958         MPI_Cart_coords(cr->mpi_comm_mysim,cr->sim_nodeid,DIM,dd->ci);
5959
5960         if (fplog)
5961         {
5962             fprintf(fplog,"Cartesian nodeid %d, coordinates %d %d %d\n\n",
5963                     cr->sim_nodeid,dd->ci[XX],dd->ci[YY],dd->ci[ZZ]);
5964         }
5965
5966         if (dd->ci[comm->cartpmedim] < dd->nc[comm->cartpmedim])
5967         {
5968             cr->duty = DUTY_PP;
5969         }
5970         if (cr->npmenodes == 0 ||
5971             dd->ci[comm->cartpmedim] >= dd->nc[comm->cartpmedim])
5972         {
5973             cr->duty = DUTY_PME;
5974         }
5975
5976         /* Split the sim communicator into PP and PME only nodes */
5977         MPI_Comm_split(cr->mpi_comm_mysim,
5978                        cr->duty,
5979                        dd_index(comm->ntot,dd->ci),
5980                        &cr->mpi_comm_mygroup);
5981     }
5982     else
5983     {
5984         switch (dd_node_order)
5985         {
5986         case ddnoPP_PME:
5987             if (fplog)
5988             {
5989                 fprintf(fplog,"Order of the nodes: PP first, PME last\n");
5990             }
5991             break;
5992         case ddnoINTERLEAVE:
5993             /* Interleave the PP-only and PME-only nodes,
5994              * as on clusters with dual-core machines this will double
5995              * the communication bandwidth of the PME processes
5996              * and thus speed up the PP <-> PME and inter PME communication.
5997              */
5998             if (fplog)
5999             {
6000                 fprintf(fplog,"Interleaving PP and PME nodes\n");
6001             }
6002             comm->pmenodes = dd_pmenodes(cr);
6003             break;
6004         case ddnoCARTESIAN:
6005             break;
6006         default:
6007             gmx_fatal(FARGS,"Unknown dd_node_order=%d",dd_node_order);
6008         }
6009
6010         if (dd_simnode2pmenode(cr,cr->sim_nodeid) == -1)
6011         {
6012             cr->duty = DUTY_PME;
6013         }
6014         else
6015         {
6016             cr->duty = DUTY_PP;
6017         }
6018
6019         /* Split the sim communicator into PP and PME only nodes */
6020         MPI_Comm_split(cr->mpi_comm_mysim,
6021                        cr->duty,
6022                        cr->nodeid,
6023                        &cr->mpi_comm_mygroup);
6024         MPI_Comm_rank(cr->mpi_comm_mygroup,&cr->nodeid);
6025     }
6026 #endif
6027
6028     if (fplog)
6029     {
6030         fprintf(fplog,"This is a %s only node\n\n",
6031                 (cr->duty & DUTY_PP) ? "particle-particle" : "PME-mesh");
6032     }
6033 }
6034
6035 void make_dd_communicators(FILE *fplog,t_commrec *cr,int dd_node_order)
6036 {
6037     gmx_domdec_t *dd;
6038     gmx_domdec_comm_t *comm;
6039     int CartReorder;
6040
6041     dd = cr->dd;
6042     comm = dd->comm;
6043
6044     copy_ivec(dd->nc,comm->ntot);
6045
6046     comm->bCartesianPP = (dd_node_order == ddnoCARTESIAN);
6047     comm->bCartesianPP_PME = FALSE;
6048
6049     /* Reorder the nodes by default. This might change the MPI ranks.
6050      * Real reordering is only supported on very few architectures,
6051      * Blue Gene is one of them.
6052      */
6053     CartReorder = (getenv("GMX_NO_CART_REORDER") == NULL);
6054
6055     if (cr->npmenodes > 0)
6056     {
6057         /* Split the communicator into a PP and PME part */
6058         split_communicator(fplog,cr,dd_node_order,CartReorder);
6059         if (comm->bCartesianPP_PME)
6060         {
6061             /* We (possibly) reordered the nodes in split_communicator,
6062              * so it is no longer required in make_pp_communicator.
6063              */
6064             CartReorder = FALSE;
6065         }
6066     }
6067     else
6068     {
6069         /* All nodes do PP and PME */
6070 #ifdef GMX_MPI
6071         /* We do not require separate communicators */
6072         cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
6073 #endif
6074     }
6075
6076     if (cr->duty & DUTY_PP)
6077     {
6078         /* Copy or make a new PP communicator */
6079         make_pp_communicator(fplog,cr,CartReorder);
6080     }
6081     else
6082     {
6083         receive_ddindex2simnodeid(cr);
6084     }
6085
6086     if (!(cr->duty & DUTY_PME))
6087     {
6088         /* Set up the commnuication to our PME node */
6089         dd->pme_nodeid = dd_simnode2pmenode(cr,cr->sim_nodeid);
6090         dd->pme_receive_vir_ener = receive_vir_ener(cr);
6091         if (debug)
6092         {
6093             fprintf(debug,"My pme_nodeid %d receive ener %d\n",
6094                     dd->pme_nodeid,dd->pme_receive_vir_ener);
6095         }
6096     }
6097     else
6098     {
6099         dd->pme_nodeid = -1;
6100     }
6101
6102     if (DDMASTER(dd))
6103     {
6104         dd->ma = init_gmx_domdec_master_t(dd,
6105                                           comm->cgs_gl.nr,
6106                                           comm->cgs_gl.index[comm->cgs_gl.nr]);
6107     }
6108 }
6109
6110 static real *get_slb_frac(FILE *fplog,const char *dir,int nc,const char *size_string)
6111 {
6112     real *slb_frac,tot;
6113     int  i,n;
6114     double dbl;
6115
6116     slb_frac = NULL;
6117     if (nc > 1 && size_string != NULL)
6118     {
6119         if (fplog)
6120         {
6121             fprintf(fplog,"Using static load balancing for the %s direction\n",
6122                     dir);
6123         }
6124         snew(slb_frac,nc);
6125         tot = 0;
6126         for (i=0; i<nc; i++)
6127         {
6128             dbl = 0;
6129             sscanf(size_string,"%lf%n",&dbl,&n);
6130             if (dbl == 0)
6131             {
6132                 gmx_fatal(FARGS,"Incorrect or not enough DD cell size entries for direction %s: '%s'",dir,size_string);
6133             }
6134             slb_frac[i] = dbl;
6135             size_string += n;
6136             tot += slb_frac[i];
6137         }
6138         /* Normalize */
6139         if (fplog)
6140         {
6141             fprintf(fplog,"Relative cell sizes:");
6142         }
6143         for (i=0; i<nc; i++)
6144         {
6145             slb_frac[i] /= tot;
6146             if (fplog)
6147             {
6148                 fprintf(fplog," %5.3f",slb_frac[i]);
6149             }
6150         }
6151         if (fplog)
6152         {
6153             fprintf(fplog,"\n");
6154         }
6155     }
6156
6157     return slb_frac;
6158 }
6159
6160 static int multi_body_bondeds_count(gmx_mtop_t *mtop)
6161 {
6162     int n,nmol,ftype;
6163     gmx_mtop_ilistloop_t iloop;
6164     t_ilist *il;
6165
6166     n = 0;
6167     iloop = gmx_mtop_ilistloop_init(mtop);
6168     while (gmx_mtop_ilistloop_next(iloop,&il,&nmol))
6169     {
6170         for(ftype=0; ftype<F_NRE; ftype++)
6171         {
6172             if ((interaction_function[ftype].flags & IF_BOND) &&
6173                 NRAL(ftype) >  2)
6174             {
6175                 n += nmol*il[ftype].nr/(1 + NRAL(ftype));
6176             }
6177         }
6178   }
6179
6180   return n;
6181 }
6182
6183 static int dd_nst_env(FILE *fplog,const char *env_var,int def)
6184 {
6185     char *val;
6186     int  nst;
6187
6188     nst = def;
6189     val = getenv(env_var);
6190     if (val)
6191     {
6192         if (sscanf(val,"%d",&nst) <= 0)
6193         {
6194             nst = 1;
6195         }
6196         if (fplog)
6197         {
6198             fprintf(fplog,"Found env.var. %s = %s, using value %d\n",
6199                     env_var,val,nst);
6200         }
6201     }
6202
6203     return nst;
6204 }
6205
6206 static void dd_warning(t_commrec *cr,FILE *fplog,const char *warn_string)
6207 {
6208     if (MASTER(cr))
6209     {
6210         fprintf(stderr,"\n%s\n",warn_string);
6211     }
6212     if (fplog)
6213     {
6214         fprintf(fplog,"\n%s\n",warn_string);
6215     }
6216 }
6217
6218 static void check_dd_restrictions(t_commrec *cr,gmx_domdec_t *dd,
6219                                   t_inputrec *ir,FILE *fplog)
6220 {
6221     if (ir->ePBC == epbcSCREW &&
6222         (dd->nc[XX] == 1 || dd->nc[YY] > 1 || dd->nc[ZZ] > 1))
6223     {
6224         gmx_fatal(FARGS,"With pbc=%s can only do domain decomposition in the x-direction",epbc_names[ir->ePBC]);
6225     }
6226
6227     if (ir->ns_type == ensSIMPLE)
6228     {
6229         gmx_fatal(FARGS,"Domain decomposition does not support simple neighbor searching, use grid searching or use particle decomposition");
6230     }
6231
6232     if (ir->nstlist == 0)
6233     {
6234         gmx_fatal(FARGS,"Domain decomposition does not work with nstlist=0");
6235     }
6236
6237     if (ir->comm_mode == ecmANGULAR && ir->ePBC != epbcNONE)
6238     {
6239         dd_warning(cr,fplog,"comm-mode angular will give incorrect results when the comm group partially crosses a periodic boundary");
6240     }
6241 }
6242
6243 static real average_cellsize_min(gmx_domdec_t *dd,gmx_ddbox_t *ddbox)
6244 {
6245     int  di,d;
6246     real r;
6247
6248     r = ddbox->box_size[XX];
6249     for(di=0; di<dd->ndim; di++)
6250     {
6251         d = dd->dim[di];
6252         /* Check using the initial average cell size */
6253         r = min(r,ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
6254     }
6255
6256     return r;
6257 }
6258
6259 static int check_dlb_support(FILE *fplog,t_commrec *cr,
6260                              const char *dlb_opt,gmx_bool bRecordLoad,
6261                              unsigned long Flags,t_inputrec *ir)
6262 {
6263     gmx_domdec_t *dd;
6264     int  eDLB=-1;
6265     char buf[STRLEN];
6266
6267     switch (dlb_opt[0])
6268     {
6269     case 'a': eDLB = edlbAUTO; break;
6270     case 'n': eDLB = edlbNO;   break;
6271     case 'y': eDLB = edlbYES;  break;
6272     default: gmx_incons("Unknown dlb_opt");
6273     }
6274
6275     if (Flags & MD_RERUN)
6276     {
6277         return edlbNO;
6278     }
6279
6280     if (!EI_DYNAMICS(ir->eI))
6281     {
6282         if (eDLB == edlbYES)
6283         {
6284             sprintf(buf,"NOTE: dynamic load balancing is only supported with dynamics, not with integrator '%s'\n",EI(ir->eI));
6285             dd_warning(cr,fplog,buf);
6286         }
6287
6288         return edlbNO;
6289     }
6290
6291     if (!bRecordLoad)
6292     {
6293         dd_warning(cr,fplog,"NOTE: Cycle counting is not supported on this architecture, will not use dynamic load balancing\n");
6294
6295         return edlbNO;
6296     }
6297
6298     if (Flags & MD_REPRODUCIBLE)
6299     {
6300         switch (eDLB)
6301         {
6302                         case edlbNO:
6303                                 break;
6304                         case edlbAUTO:
6305                                 dd_warning(cr,fplog,"NOTE: reproducibility requested, will not use dynamic load balancing\n");
6306                                 eDLB = edlbNO;
6307                                 break;
6308                         case edlbYES:
6309                                 dd_warning(cr,fplog,"WARNING: reproducibility requested with dynamic load balancing, the simulation will NOT be binary reproducible\n");
6310                                 break;
6311                         default:
6312                                 gmx_fatal(FARGS,"Death horror: undefined case (%d) for load balancing choice",eDLB);
6313                                 break;
6314         }
6315     }
6316
6317     return eDLB;
6318 }
6319
6320 static void set_dd_dim(FILE *fplog,gmx_domdec_t *dd)
6321 {
6322     int dim;
6323
6324     dd->ndim = 0;
6325     if (getenv("GMX_DD_ORDER_ZYX") != NULL)
6326     {
6327         /* Decomposition order z,y,x */
6328         if (fplog)
6329         {
6330             fprintf(fplog,"Using domain decomposition order z, y, x\n");
6331         }
6332         for(dim=DIM-1; dim>=0; dim--)
6333         {
6334             if (dd->nc[dim] > 1)
6335             {
6336                 dd->dim[dd->ndim++] = dim;
6337             }
6338         }
6339     }
6340     else
6341     {
6342         /* Decomposition order x,y,z */
6343         for(dim=0; dim<DIM; dim++)
6344         {
6345             if (dd->nc[dim] > 1)
6346             {
6347                 dd->dim[dd->ndim++] = dim;
6348             }
6349         }
6350     }
6351 }
6352
6353 static gmx_domdec_comm_t *init_dd_comm()
6354 {
6355     gmx_domdec_comm_t *comm;
6356     int  i;
6357
6358     snew(comm,1);
6359     snew(comm->cggl_flag,DIM*2);
6360     snew(comm->cgcm_state,DIM*2);
6361     for(i=0; i<DIM*2; i++)
6362     {
6363         comm->cggl_flag_nalloc[i]  = 0;
6364         comm->cgcm_state_nalloc[i] = 0;
6365     }
6366
6367     comm->nalloc_int = 0;
6368     comm->buf_int    = NULL;
6369
6370     vec_rvec_init(&comm->vbuf);
6371
6372     comm->n_load_have    = 0;
6373     comm->n_load_collect = 0;
6374
6375     for(i=0; i<ddnatNR-ddnatZONE; i++)
6376     {
6377         comm->sum_nat[i] = 0;
6378     }
6379     comm->ndecomp = 0;
6380     comm->nload   = 0;
6381     comm->load_step = 0;
6382     comm->load_sum  = 0;
6383     comm->load_max  = 0;
6384     clear_ivec(comm->load_lim);
6385     comm->load_mdf  = 0;
6386     comm->load_pme  = 0;
6387
6388     return comm;
6389 }
6390
6391 gmx_domdec_t *init_domain_decomposition(FILE *fplog,t_commrec *cr,
6392                                         unsigned long Flags,
6393                                         ivec nc,
6394                                         real comm_distance_min,real rconstr,
6395                                         const char *dlb_opt,real dlb_scale,
6396                                         const char *sizex,const char *sizey,const char *sizez,
6397                                         gmx_mtop_t *mtop,t_inputrec *ir,
6398                                         matrix box,rvec *x,
6399                                         gmx_ddbox_t *ddbox,
6400                                         int *npme_x,int *npme_y)
6401 {
6402     gmx_domdec_t *dd;
6403     gmx_domdec_comm_t *comm;
6404     int  recload;
6405     int  d,i,j;
6406     real r_2b,r_mb,r_bonded=-1,r_bonded_limit=-1,limit,acs;
6407     gmx_bool bC;
6408     char buf[STRLEN];
6409
6410     if (fplog)
6411     {
6412         fprintf(fplog,
6413                 "\nInitializing Domain Decomposition on %d nodes\n",cr->nnodes);
6414     }
6415
6416     snew(dd,1);
6417
6418     dd->comm = init_dd_comm();
6419     comm = dd->comm;
6420     snew(comm->cggl_flag,DIM*2);
6421     snew(comm->cgcm_state,DIM*2);
6422
6423     dd->npbcdim   = ePBC2npbcdim(ir->ePBC);
6424     dd->bScrewPBC = (ir->ePBC == epbcSCREW);
6425
6426     dd->bSendRecv2      = dd_nst_env(fplog,"GMX_DD_SENDRECV2",0);
6427     comm->dlb_scale_lim = dd_nst_env(fplog,"GMX_DLB_MAX",10);
6428     comm->eFlop         = dd_nst_env(fplog,"GMX_DLB_FLOP",0);
6429     recload             = dd_nst_env(fplog,"GMX_DD_LOAD",1);
6430     comm->nstSortCG     = dd_nst_env(fplog,"GMX_DD_SORT",1);
6431     comm->nstDDDump     = dd_nst_env(fplog,"GMX_DD_DUMP",0);
6432     comm->nstDDDumpGrid = dd_nst_env(fplog,"GMX_DD_DUMP_GRID",0);
6433     comm->DD_debug      = dd_nst_env(fplog,"GMX_DD_DEBUG",0);
6434
6435     dd->pme_recv_f_alloc = 0;
6436     dd->pme_recv_f_buf = NULL;
6437
6438     if (dd->bSendRecv2 && fplog)
6439     {
6440         fprintf(fplog,"Will use two sequential MPI_Sendrecv calls instead of two simultaneous non-blocking MPI_Irecv and MPI_Isend pairs for constraint and vsite communication\n");
6441     }
6442     if (comm->eFlop)
6443     {
6444         if (fplog)
6445         {
6446             fprintf(fplog,"Will load balance based on FLOP count\n");
6447         }
6448         if (comm->eFlop > 1)
6449         {
6450             srand(1+cr->nodeid);
6451         }
6452         comm->bRecordLoad = TRUE;
6453     }
6454     else
6455     {
6456         comm->bRecordLoad = (wallcycle_have_counter() && recload > 0);
6457
6458     }
6459
6460     comm->eDLB = check_dlb_support(fplog,cr,dlb_opt,comm->bRecordLoad,Flags,ir);
6461
6462     comm->bDynLoadBal = (comm->eDLB == edlbYES);
6463     if (fplog)
6464     {
6465         fprintf(fplog,"Dynamic load balancing: %s\n",edlb_names[comm->eDLB]);
6466     }
6467     dd->bGridJump = comm->bDynLoadBal;
6468
6469     if (comm->nstSortCG)
6470     {
6471         if (fplog)
6472         {
6473             if (comm->nstSortCG == 1)
6474             {
6475                 fprintf(fplog,"Will sort the charge groups at every domain (re)decomposition\n");
6476             }
6477             else
6478             {
6479                 fprintf(fplog,"Will sort the charge groups every %d steps\n",
6480                         comm->nstSortCG);
6481             }
6482         }
6483         snew(comm->sort,1);
6484     }
6485     else
6486     {
6487         if (fplog)
6488         {
6489             fprintf(fplog,"Will not sort the charge groups\n");
6490         }
6491     }
6492
6493     comm->bCGs = (ncg_mtop(mtop) < mtop->natoms);
6494
6495     comm->bInterCGBondeds = (ncg_mtop(mtop) > mtop->mols.nr);
6496     if (comm->bInterCGBondeds)
6497     {
6498         comm->bInterCGMultiBody = (multi_body_bondeds_count(mtop) > 0);
6499     }
6500     else
6501     {
6502         comm->bInterCGMultiBody = FALSE;
6503     }
6504
6505     dd->bInterCGcons    = inter_charge_group_constraints(mtop);
6506     dd->bInterCGsettles = inter_charge_group_settles(mtop);
6507
6508     if (ir->rlistlong == 0)
6509     {
6510         /* Set the cut-off to some very large value,
6511          * so we don't need if statements everywhere in the code.
6512          * We use sqrt, since the cut-off is squared in some places.
6513          */
6514         comm->cutoff   = GMX_CUTOFF_INF;
6515     }
6516     else
6517     {
6518         comm->cutoff   = ir->rlistlong;
6519     }
6520     comm->cutoff_mbody = 0;
6521
6522     comm->cellsize_limit = 0;
6523     comm->bBondComm = FALSE;
6524
6525     if (comm->bInterCGBondeds)
6526     {
6527         if (comm_distance_min > 0)
6528         {
6529             comm->cutoff_mbody = comm_distance_min;
6530             if (Flags & MD_DDBONDCOMM)
6531             {
6532                 comm->bBondComm = (comm->cutoff_mbody > comm->cutoff);
6533             }
6534             else
6535             {
6536                 comm->cutoff = max(comm->cutoff,comm->cutoff_mbody);
6537             }
6538             r_bonded_limit = comm->cutoff_mbody;
6539         }
6540         else if (ir->bPeriodicMols)
6541         {
6542             /* Can not easily determine the required cut-off */
6543             dd_warning(cr,fplog,"NOTE: Periodic molecules are present in this system. Because of this, the domain decomposition algorithm cannot easily determine the minimum cell size that it requires for treating bonded interactions. Instead, domain decomposition will assume that half the non-bonded cut-off will be a suitable lower bound.\n");
6544             comm->cutoff_mbody = comm->cutoff/2;
6545             r_bonded_limit = comm->cutoff_mbody;
6546         }
6547         else
6548         {
6549             if (MASTER(cr))
6550             {
6551                 dd_bonded_cg_distance(fplog,dd,mtop,ir,x,box,
6552                                       Flags & MD_DDBONDCHECK,&r_2b,&r_mb);
6553             }
6554             gmx_bcast(sizeof(r_2b),&r_2b,cr);
6555             gmx_bcast(sizeof(r_mb),&r_mb,cr);
6556
6557             /* We use an initial margin of 10% for the minimum cell size,
6558              * except when we are just below the non-bonded cut-off.
6559              */
6560             if (Flags & MD_DDBONDCOMM)
6561             {
6562                 if (max(r_2b,r_mb) > comm->cutoff)
6563                 {
6564                     r_bonded       = max(r_2b,r_mb);
6565                     r_bonded_limit = 1.1*r_bonded;
6566                     comm->bBondComm = TRUE;
6567                 }
6568                 else
6569                 {
6570                     r_bonded       = r_mb;
6571                     r_bonded_limit = min(1.1*r_bonded,comm->cutoff);
6572                 }
6573                 /* We determine cutoff_mbody later */
6574             }
6575             else
6576             {
6577                 /* No special bonded communication,
6578                  * simply increase the DD cut-off.
6579                  */
6580                 r_bonded_limit     = 1.1*max(r_2b,r_mb);
6581                 comm->cutoff_mbody = r_bonded_limit;
6582                 comm->cutoff       = max(comm->cutoff,comm->cutoff_mbody);
6583             }
6584         }
6585         comm->cellsize_limit = max(comm->cellsize_limit,r_bonded_limit);
6586         if (fplog)
6587         {
6588             fprintf(fplog,
6589                     "Minimum cell size due to bonded interactions: %.3f nm\n",
6590                     comm->cellsize_limit);
6591         }
6592     }
6593
6594     if (dd->bInterCGcons && rconstr <= 0)
6595     {
6596         /* There is a cell size limit due to the constraints (P-LINCS) */
6597         rconstr = constr_r_max(fplog,mtop,ir);
6598         if (fplog)
6599         {
6600             fprintf(fplog,
6601                     "Estimated maximum distance required for P-LINCS: %.3f nm\n",
6602                     rconstr);
6603             if (rconstr > comm->cellsize_limit)
6604             {
6605                 fprintf(fplog,"This distance will limit the DD cell size, you can override this with -rcon\n");
6606             }
6607         }
6608     }
6609     else if (rconstr > 0 && fplog)
6610     {
6611         /* Here we do not check for dd->bInterCGcons,
6612          * because one can also set a cell size limit for virtual sites only
6613          * and at this point we don't know yet if there are intercg v-sites.
6614          */
6615         fprintf(fplog,
6616                 "User supplied maximum distance required for P-LINCS: %.3f nm\n",
6617                 rconstr);
6618     }
6619     comm->cellsize_limit = max(comm->cellsize_limit,rconstr);
6620
6621     comm->cgs_gl = gmx_mtop_global_cgs(mtop);
6622
6623     if (nc[XX] > 0)
6624     {
6625         copy_ivec(nc,dd->nc);
6626         set_dd_dim(fplog,dd);
6627         set_ddbox_cr(cr,&dd->nc,ir,box,&comm->cgs_gl,x,ddbox);
6628
6629         if (cr->npmenodes == -1)
6630         {
6631             cr->npmenodes = 0;
6632         }
6633         acs = average_cellsize_min(dd,ddbox);
6634         if (acs < comm->cellsize_limit)
6635         {
6636             if (fplog)
6637             {
6638                 fprintf(fplog,"ERROR: The initial cell size (%f) is smaller than the cell size limit (%f)\n",acs,comm->cellsize_limit);
6639             }
6640             gmx_fatal_collective(FARGS,cr,NULL,
6641                                  "The initial cell size (%f) is smaller than the cell size limit (%f), change options -dd, -rdd or -rcon, see the log file for details",
6642                                  acs,comm->cellsize_limit);
6643         }
6644     }
6645     else
6646     {
6647         set_ddbox_cr(cr,NULL,ir,box,&comm->cgs_gl,x,ddbox);
6648
6649         /* We need to choose the optimal DD grid and possibly PME nodes */
6650         limit = dd_choose_grid(fplog,cr,dd,ir,mtop,box,ddbox,
6651                                comm->eDLB!=edlbNO,dlb_scale,
6652                                comm->cellsize_limit,comm->cutoff,
6653                                comm->bInterCGBondeds,comm->bInterCGMultiBody);
6654
6655         if (dd->nc[XX] == 0)
6656         {
6657             bC = (dd->bInterCGcons && rconstr > r_bonded_limit);
6658             sprintf(buf,"Change the number of nodes or mdrun option %s%s%s",
6659                     !bC ? "-rdd" : "-rcon",
6660                     comm->eDLB!=edlbNO ? " or -dds" : "",
6661                     bC ? " or your LINCS settings" : "");
6662
6663             gmx_fatal_collective(FARGS,cr,NULL,
6664                                  "There is no domain decomposition for %d nodes that is compatible with the given box and a minimum cell size of %g nm\n"
6665                                  "%s\n"
6666                                  "Look in the log file for details on the domain decomposition",
6667                                  cr->nnodes-cr->npmenodes,limit,buf);
6668         }
6669         set_dd_dim(fplog,dd);
6670     }
6671
6672     if (fplog)
6673     {
6674         fprintf(fplog,
6675                 "Domain decomposition grid %d x %d x %d, separate PME nodes %d\n",
6676                 dd->nc[XX],dd->nc[YY],dd->nc[ZZ],cr->npmenodes);
6677     }
6678
6679     dd->nnodes = dd->nc[XX]*dd->nc[YY]*dd->nc[ZZ];
6680     if (cr->nnodes - dd->nnodes != cr->npmenodes)
6681     {
6682         gmx_fatal_collective(FARGS,cr,NULL,
6683                              "The size of the domain decomposition grid (%d) does not match the number of nodes (%d). The total number of nodes is %d",
6684                              dd->nnodes,cr->nnodes - cr->npmenodes,cr->nnodes);
6685     }
6686     if (cr->npmenodes > dd->nnodes)
6687     {
6688         gmx_fatal_collective(FARGS,cr,NULL,
6689                              "The number of separate PME nodes (%d) is larger than the number of PP nodes (%d), this is not supported.",cr->npmenodes,dd->nnodes);
6690     }
6691     if (cr->npmenodes > 0)
6692     {
6693         comm->npmenodes = cr->npmenodes;
6694     }
6695     else
6696     {
6697         comm->npmenodes = dd->nnodes;
6698     }
6699
6700     if (EEL_PME(ir->coulombtype))
6701     {
6702         /* The following choices should match those
6703          * in comm_cost_est in domdec_setup.c.
6704          * Note that here the checks have to take into account
6705          * that the decomposition might occur in a different order than xyz
6706          * (for instance through the env.var. GMX_DD_ORDER_ZYX),
6707          * in which case they will not match those in comm_cost_est,
6708          * but since that is mainly for testing purposes that's fine.
6709          */
6710         if (dd->ndim >= 2 && dd->dim[0] == XX && dd->dim[1] == YY &&
6711             comm->npmenodes > dd->nc[XX] && comm->npmenodes % dd->nc[XX] == 0 &&
6712             getenv("GMX_PMEONEDD") == NULL)
6713         {
6714             comm->npmedecompdim = 2;
6715             comm->npmenodes_x   = dd->nc[XX];
6716             comm->npmenodes_y   = comm->npmenodes/comm->npmenodes_x;
6717         }
6718         else
6719         {
6720             /* In case nc is 1 in both x and y we could still choose to
6721              * decompose pme in y instead of x, but we use x for simplicity.
6722              */
6723             comm->npmedecompdim = 1;
6724             if (dd->dim[0] == YY)
6725             {
6726                 comm->npmenodes_x = 1;
6727                 comm->npmenodes_y = comm->npmenodes;
6728             }
6729             else
6730             {
6731                 comm->npmenodes_x = comm->npmenodes;
6732                 comm->npmenodes_y = 1;
6733             }
6734         }
6735         if (fplog)
6736         {
6737             fprintf(fplog,"PME domain decomposition: %d x %d x %d\n",
6738                     comm->npmenodes_x,comm->npmenodes_y,1);
6739         }
6740     }
6741     else
6742     {
6743         comm->npmedecompdim = 0;
6744         comm->npmenodes_x   = 0;
6745         comm->npmenodes_y   = 0;
6746     }
6747
6748     /* Technically we don't need both of these,
6749      * but it simplifies code not having to recalculate it.
6750      */
6751     *npme_x = comm->npmenodes_x;
6752     *npme_y = comm->npmenodes_y;
6753
6754     snew(comm->slb_frac,DIM);
6755     if (comm->eDLB == edlbNO)
6756     {
6757         comm->slb_frac[XX] = get_slb_frac(fplog,"x",dd->nc[XX],sizex);
6758         comm->slb_frac[YY] = get_slb_frac(fplog,"y",dd->nc[YY],sizey);
6759         comm->slb_frac[ZZ] = get_slb_frac(fplog,"z",dd->nc[ZZ],sizez);
6760     }
6761
6762     if (comm->bInterCGBondeds && comm->cutoff_mbody == 0)
6763     {
6764         if (comm->bBondComm || comm->eDLB != edlbNO)
6765         {
6766             /* Set the bonded communication distance to halfway
6767              * the minimum and the maximum,
6768              * since the extra communication cost is nearly zero.
6769              */
6770             acs = average_cellsize_min(dd,ddbox);
6771             comm->cutoff_mbody = 0.5*(r_bonded + acs);
6772             if (comm->eDLB != edlbNO)
6773             {
6774                 /* Check if this does not limit the scaling */
6775                 comm->cutoff_mbody = min(comm->cutoff_mbody,dlb_scale*acs);
6776             }
6777             if (!comm->bBondComm)
6778             {
6779                 /* Without bBondComm do not go beyond the n.b. cut-off */
6780                 comm->cutoff_mbody = min(comm->cutoff_mbody,comm->cutoff);
6781                 if (comm->cellsize_limit >= comm->cutoff)
6782                 {
6783                     /* We don't loose a lot of efficieny
6784                      * when increasing it to the n.b. cut-off.
6785                      * It can even be slightly faster, because we need
6786                      * less checks for the communication setup.
6787                      */
6788                     comm->cutoff_mbody = comm->cutoff;
6789                 }
6790             }
6791             /* Check if we did not end up below our original limit */
6792             comm->cutoff_mbody = max(comm->cutoff_mbody,r_bonded_limit);
6793
6794             if (comm->cutoff_mbody > comm->cellsize_limit)
6795             {
6796                 comm->cellsize_limit = comm->cutoff_mbody;
6797             }
6798         }
6799         /* Without DLB and cutoff_mbody<cutoff, cutoff_mbody is dynamic */
6800     }
6801
6802     if (debug)
6803     {
6804         fprintf(debug,"Bonded atom communication beyond the cut-off: %d\n"
6805                 "cellsize limit %f\n",
6806                 comm->bBondComm,comm->cellsize_limit);
6807     }
6808
6809     if (MASTER(cr))
6810     {
6811         check_dd_restrictions(cr,dd,ir,fplog);
6812     }
6813
6814     comm->partition_step = INT_MIN;
6815     dd->ddp_count = 0;
6816
6817     clear_dd_cycle_counts(dd);
6818
6819     return dd;
6820 }
6821
6822 static void set_dlb_limits(gmx_domdec_t *dd)
6823
6824 {
6825     int d;
6826
6827     for(d=0; d<dd->ndim; d++)
6828     {
6829         dd->comm->cd[d].np = dd->comm->cd[d].np_dlb;
6830         dd->comm->cellsize_min[dd->dim[d]] =
6831             dd->comm->cellsize_min_dlb[dd->dim[d]];
6832     }
6833 }
6834
6835
6836 static void turn_on_dlb(FILE *fplog,t_commrec *cr,gmx_large_int_t step)
6837 {
6838     gmx_domdec_t *dd;
6839     gmx_domdec_comm_t *comm;
6840     real cellsize_min;
6841     int  d,nc,i;
6842     char buf[STRLEN];
6843
6844     dd = cr->dd;
6845     comm = dd->comm;
6846
6847     if (fplog)
6848     {
6849         fprintf(fplog,"At step %s the performance loss due to force load imbalance is %.1f %%\n",gmx_step_str(step,buf),dd_force_imb_perf_loss(dd)*100);
6850     }
6851
6852     cellsize_min = comm->cellsize_min[dd->dim[0]];
6853     for(d=1; d<dd->ndim; d++)
6854     {
6855         cellsize_min = min(cellsize_min,comm->cellsize_min[dd->dim[d]]);
6856     }
6857
6858     if (cellsize_min < comm->cellsize_limit*1.05)
6859     {
6860         dd_warning(cr,fplog,"NOTE: the minimum cell size is smaller than 1.05 times the cell size limit, will not turn on dynamic load balancing\n");
6861
6862         /* Change DLB from "auto" to "no". */
6863         comm->eDLB = edlbNO;
6864
6865         return;
6866     }
6867
6868     dd_warning(cr,fplog,"NOTE: Turning on dynamic load balancing\n");
6869     comm->bDynLoadBal = TRUE;
6870     dd->bGridJump = TRUE;
6871
6872     set_dlb_limits(dd);
6873
6874     /* We can set the required cell size info here,
6875      * so we do not need to communicate this.
6876      * The grid is completely uniform.
6877      */
6878     for(d=0; d<dd->ndim; d++)
6879     {
6880         if (comm->root[d])
6881         {
6882             comm->load[d].sum_m = comm->load[d].sum;
6883
6884             nc = dd->nc[dd->dim[d]];
6885             for(i=0; i<nc; i++)
6886             {
6887                 comm->root[d]->cell_f[i]    = i/(real)nc;
6888                 if (d > 0)
6889                 {
6890                     comm->root[d]->cell_f_max0[i] =  i   /(real)nc;
6891                     comm->root[d]->cell_f_min1[i] = (i+1)/(real)nc;
6892                 }
6893             }
6894             comm->root[d]->cell_f[nc] = 1.0;
6895         }
6896     }
6897 }
6898
6899 static char *init_bLocalCG(gmx_mtop_t *mtop)
6900 {
6901     int  ncg,cg;
6902     char *bLocalCG;
6903
6904     ncg = ncg_mtop(mtop);
6905     snew(bLocalCG,ncg);
6906     for(cg=0; cg<ncg; cg++)
6907     {
6908         bLocalCG[cg] = FALSE;
6909     }
6910
6911     return bLocalCG;
6912 }
6913
6914 void dd_init_bondeds(FILE *fplog,
6915                      gmx_domdec_t *dd,gmx_mtop_t *mtop,
6916                      gmx_vsite_t *vsite,gmx_constr_t constr,
6917                      t_inputrec *ir,gmx_bool bBCheck,cginfo_mb_t *cginfo_mb)
6918 {
6919     gmx_domdec_comm_t *comm;
6920     gmx_bool bBondComm;
6921     int  d;
6922
6923     dd_make_reverse_top(fplog,dd,mtop,vsite,constr,ir,bBCheck);
6924
6925     comm = dd->comm;
6926
6927     if (comm->bBondComm)
6928     {
6929         /* Communicate atoms beyond the cut-off for bonded interactions */
6930         comm = dd->comm;
6931
6932         comm->cglink = make_charge_group_links(mtop,dd,cginfo_mb);
6933
6934         comm->bLocalCG = init_bLocalCG(mtop);
6935     }
6936     else
6937     {
6938         /* Only communicate atoms based on cut-off */
6939         comm->cglink   = NULL;
6940         comm->bLocalCG = NULL;
6941     }
6942 }
6943
6944 static void print_dd_settings(FILE *fplog,gmx_domdec_t *dd,
6945                               t_inputrec *ir,
6946                               gmx_bool bDynLoadBal,real dlb_scale,
6947                               gmx_ddbox_t *ddbox)
6948 {
6949     gmx_domdec_comm_t *comm;
6950     int  d;
6951     ivec np;
6952     real limit,shrink;
6953     char buf[64];
6954
6955     if (fplog == NULL)
6956     {
6957         return;
6958     }
6959
6960     comm = dd->comm;
6961
6962     if (bDynLoadBal)
6963     {
6964         fprintf(fplog,"The maximum number of communication pulses is:");
6965         for(d=0; d<dd->ndim; d++)
6966         {
6967             fprintf(fplog," %c %d",dim2char(dd->dim[d]),comm->cd[d].np_dlb);
6968         }
6969         fprintf(fplog,"\n");
6970         fprintf(fplog,"The minimum size for domain decomposition cells is %.3f nm\n",comm->cellsize_limit);
6971         fprintf(fplog,"The requested allowed shrink of DD cells (option -dds) is: %.2f\n",dlb_scale);
6972         fprintf(fplog,"The allowed shrink of domain decomposition cells is:");
6973         for(d=0; d<DIM; d++)
6974         {
6975             if (dd->nc[d] > 1)
6976             {
6977                 if (d >= ddbox->npbcdim && dd->nc[d] == 2)
6978                 {
6979                     shrink = 0;
6980                 }
6981                 else
6982                 {
6983                     shrink =
6984                         comm->cellsize_min_dlb[d]/
6985                         (ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
6986                 }
6987                 fprintf(fplog," %c %.2f",dim2char(d),shrink);
6988             }
6989         }
6990         fprintf(fplog,"\n");
6991     }
6992     else
6993     {
6994         set_dd_cell_sizes_slb(dd,ddbox,FALSE,np);
6995         fprintf(fplog,"The initial number of communication pulses is:");
6996         for(d=0; d<dd->ndim; d++)
6997         {
6998             fprintf(fplog," %c %d",dim2char(dd->dim[d]),np[dd->dim[d]]);
6999         }
7000         fprintf(fplog,"\n");
7001         fprintf(fplog,"The initial domain decomposition cell size is:");
7002         for(d=0; d<DIM; d++) {
7003             if (dd->nc[d] > 1)
7004             {
7005                 fprintf(fplog," %c %.2f nm",
7006                         dim2char(d),dd->comm->cellsize_min[d]);
7007             }
7008         }
7009         fprintf(fplog,"\n\n");
7010     }
7011
7012     if (comm->bInterCGBondeds || dd->vsite_comm || dd->constraint_comm)
7013     {
7014         fprintf(fplog,"The maximum allowed distance for charge groups involved in interactions is:\n");
7015         fprintf(fplog,"%40s  %-7s %6.3f nm\n",
7016                 "non-bonded interactions","",comm->cutoff);
7017
7018         if (bDynLoadBal)
7019         {
7020             limit = dd->comm->cellsize_limit;
7021         }
7022         else
7023         {
7024             if (dynamic_dd_box(ddbox,ir))
7025             {
7026                 fprintf(fplog,"(the following are initial values, they could change due to box deformation)\n");
7027             }
7028             limit = dd->comm->cellsize_min[XX];
7029             for(d=1; d<DIM; d++)
7030             {
7031                 limit = min(limit,dd->comm->cellsize_min[d]);
7032             }
7033         }
7034
7035         if (comm->bInterCGBondeds)
7036         {
7037             fprintf(fplog,"%40s  %-7s %6.3f nm\n",
7038                     "two-body bonded interactions","(-rdd)",
7039                     max(comm->cutoff,comm->cutoff_mbody));
7040             fprintf(fplog,"%40s  %-7s %6.3f nm\n",
7041                     "multi-body bonded interactions","(-rdd)",
7042                     (comm->bBondComm || dd->bGridJump) ? comm->cutoff_mbody : min(comm->cutoff,limit));
7043         }
7044         if (dd->vsite_comm)
7045         {
7046             fprintf(fplog,"%40s  %-7s %6.3f nm\n",
7047                     "virtual site constructions","(-rcon)",limit);
7048         }
7049         if (dd->constraint_comm)
7050         {
7051             sprintf(buf,"atoms separated by up to %d constraints",
7052                     1+ir->nProjOrder);
7053             fprintf(fplog,"%40s  %-7s %6.3f nm\n",
7054                     buf,"(-rcon)",limit);
7055         }
7056         fprintf(fplog,"\n");
7057     }
7058
7059     fflush(fplog);
7060 }
7061
7062 static void set_cell_limits_dlb(gmx_domdec_t *dd,
7063                                 real dlb_scale,
7064                                 const t_inputrec *ir,
7065                                 const gmx_ddbox_t *ddbox)
7066 {
7067     gmx_domdec_comm_t *comm;
7068     int  d,dim,npulse,npulse_d_max,npulse_d;
7069     gmx_bool bNoCutOff;
7070
7071     comm = dd->comm;
7072
7073     bNoCutOff = (ir->rvdw == 0 || ir->rcoulomb == 0);
7074
7075     /* Determine the maximum number of comm. pulses in one dimension */
7076
7077     comm->cellsize_limit = max(comm->cellsize_limit,comm->cutoff_mbody);
7078
7079     /* Determine the maximum required number of grid pulses */
7080     if (comm->cellsize_limit >= comm->cutoff)
7081     {
7082         /* Only a single pulse is required */
7083         npulse = 1;
7084     }
7085     else if (!bNoCutOff && comm->cellsize_limit > 0)
7086     {
7087         /* We round down slightly here to avoid overhead due to the latency
7088          * of extra communication calls when the cut-off
7089          * would be only slightly longer than the cell size.
7090          * Later cellsize_limit is redetermined,
7091          * so we can not miss interactions due to this rounding.
7092          */
7093         npulse = (int)(0.96 + comm->cutoff/comm->cellsize_limit);
7094     }
7095     else
7096     {
7097         /* There is no cell size limit */
7098         npulse = max(dd->nc[XX]-1,max(dd->nc[YY]-1,dd->nc[ZZ]-1));
7099     }
7100
7101     if (!bNoCutOff && npulse > 1)
7102     {
7103         /* See if we can do with less pulses, based on dlb_scale */
7104         npulse_d_max = 0;
7105         for(d=0; d<dd->ndim; d++)
7106         {
7107             dim = dd->dim[d];
7108             npulse_d = (int)(1 + dd->nc[dim]*comm->cutoff
7109                              /(ddbox->box_size[dim]*ddbox->skew_fac[dim]*dlb_scale));
7110             npulse_d_max = max(npulse_d_max,npulse_d);
7111         }
7112         npulse = min(npulse,npulse_d_max);
7113     }
7114
7115     /* This env var can override npulse */
7116     d = dd_nst_env(debug,"GMX_DD_NPULSE",0);
7117     if (d > 0)
7118     {
7119         npulse = d;
7120     }
7121
7122     comm->maxpulse = 1;
7123     comm->bVacDLBNoLimit = (ir->ePBC == epbcNONE);
7124     for(d=0; d<dd->ndim; d++)
7125     {
7126         comm->cd[d].np_dlb = min(npulse,dd->nc[dd->dim[d]]-1);
7127         comm->cd[d].np_nalloc = comm->cd[d].np_dlb;
7128         snew(comm->cd[d].ind,comm->cd[d].np_nalloc);
7129         comm->maxpulse = max(comm->maxpulse,comm->cd[d].np_dlb);
7130         if (comm->cd[d].np_dlb < dd->nc[dd->dim[d]]-1)
7131         {
7132             comm->bVacDLBNoLimit = FALSE;
7133         }
7134     }
7135
7136     /* cellsize_limit is set for LINCS in init_domain_decomposition */
7137     if (!comm->bVacDLBNoLimit)
7138     {
7139         comm->cellsize_limit = max(comm->cellsize_limit,
7140                                    comm->cutoff/comm->maxpulse);
7141     }
7142     comm->cellsize_limit = max(comm->cellsize_limit,comm->cutoff_mbody);
7143     /* Set the minimum cell size for each DD dimension */
7144     for(d=0; d<dd->ndim; d++)
7145     {
7146         if (comm->bVacDLBNoLimit ||
7147             comm->cd[d].np_dlb*comm->cellsize_limit >= comm->cutoff)
7148         {
7149             comm->cellsize_min_dlb[dd->dim[d]] = comm->cellsize_limit;
7150         }
7151         else
7152         {
7153             comm->cellsize_min_dlb[dd->dim[d]] =
7154                 comm->cutoff/comm->cd[d].np_dlb;
7155         }
7156     }
7157     if (comm->cutoff_mbody <= 0)
7158     {
7159         comm->cutoff_mbody = min(comm->cutoff,comm->cellsize_limit);
7160     }
7161     if (comm->bDynLoadBal)
7162     {
7163         set_dlb_limits(dd);
7164     }
7165 }
7166
7167 gmx_bool dd_bonded_molpbc(gmx_domdec_t *dd,int ePBC)
7168 {
7169     /* If each molecule is a single charge group
7170      * or we use domain decomposition for each periodic dimension,
7171      * we do not need to take pbc into account for the bonded interactions.
7172      */
7173     return (ePBC != epbcNONE && dd->comm->bInterCGBondeds &&
7174             !(dd->nc[XX]>1 &&
7175               dd->nc[YY]>1 &&
7176               (dd->nc[ZZ]>1 || ePBC==epbcXY)));
7177 }
7178
7179 void set_dd_parameters(FILE *fplog,gmx_domdec_t *dd,real dlb_scale,
7180                        t_inputrec *ir,t_forcerec *fr,
7181                        gmx_ddbox_t *ddbox)
7182 {
7183     gmx_domdec_comm_t *comm;
7184     int  natoms_tot;
7185     real vol_frac;
7186
7187     comm = dd->comm;
7188
7189     /* Initialize the thread data.
7190      * This can not be done in init_domain_decomposition,
7191      * as the numbers of threads is determined later.
7192      */
7193     comm->nth = gmx_omp_nthreads_get(emntDomdec);
7194     if (comm->nth > 1)
7195     {
7196         snew(comm->dth,comm->nth);
7197     }
7198
7199     if (EEL_PME(ir->coulombtype))
7200     {
7201         init_ddpme(dd,&comm->ddpme[0],0);
7202         if (comm->npmedecompdim >= 2)
7203         {
7204             init_ddpme(dd,&comm->ddpme[1],1);
7205         }
7206     }
7207     else
7208     {
7209         comm->npmenodes = 0;
7210         if (dd->pme_nodeid >= 0)
7211         {
7212             gmx_fatal_collective(FARGS,NULL,dd,
7213                                  "Can not have separate PME nodes without PME electrostatics");
7214         }
7215     }
7216
7217     if (debug)
7218     {
7219         fprintf(debug,"The DD cut-off is %f\n",comm->cutoff);
7220     }
7221     if (comm->eDLB != edlbNO)
7222     {
7223         set_cell_limits_dlb(dd,dlb_scale,ir,ddbox);
7224     }
7225
7226     print_dd_settings(fplog,dd,ir,comm->bDynLoadBal,dlb_scale,ddbox);
7227     if (comm->eDLB == edlbAUTO)
7228     {
7229         if (fplog)
7230         {
7231             fprintf(fplog,"When dynamic load balancing gets turned on, these settings will change to:\n");
7232         }
7233         print_dd_settings(fplog,dd,ir,TRUE,dlb_scale,ddbox);
7234     }
7235
7236     if (ir->ePBC == epbcNONE)
7237     {
7238         vol_frac = 1 - 1/(double)dd->nnodes;
7239     }
7240     else
7241     {
7242         vol_frac =
7243             (1 + comm_box_frac(dd->nc,comm->cutoff,ddbox))/(double)dd->nnodes;
7244     }
7245     if (debug)
7246     {
7247         fprintf(debug,"Volume fraction for all DD zones: %f\n",vol_frac);
7248     }
7249     natoms_tot = comm->cgs_gl.index[comm->cgs_gl.nr];
7250
7251     dd->ga2la = ga2la_init(natoms_tot,vol_frac*natoms_tot);
7252 }
7253
7254 gmx_bool change_dd_cutoff(t_commrec *cr,t_state *state,t_inputrec *ir,
7255                           real cutoff_req)
7256 {
7257     gmx_domdec_t *dd;
7258     gmx_ddbox_t ddbox;
7259     int d,dim,np;
7260     real inv_cell_size;
7261     int LocallyLimited;
7262
7263     dd = cr->dd;
7264
7265     set_ddbox(dd,FALSE,cr,ir,state->box,
7266               TRUE,&dd->comm->cgs_gl,state->x,&ddbox);
7267
7268     LocallyLimited = 0;
7269
7270     for(d=0; d<dd->ndim; d++)
7271     {
7272         dim = dd->dim[d];
7273
7274         inv_cell_size = DD_CELL_MARGIN*dd->nc[dim]/ddbox.box_size[dim];
7275         if (dynamic_dd_box(&ddbox,ir))
7276         {
7277             inv_cell_size *= DD_PRES_SCALE_MARGIN;
7278         }
7279
7280         np = 1 + (int)(cutoff_req*inv_cell_size*ddbox.skew_fac[dim]);
7281
7282         if (dd->comm->eDLB != edlbNO && dim < ddbox.npbcdim &&
7283             dd->comm->cd[d].np_dlb > 0)
7284         {
7285             if (np > dd->comm->cd[d].np_dlb)
7286             {
7287                 return FALSE;
7288             }
7289
7290             /* If a current local cell size is smaller than the requested
7291              * cut-off, we could still fix it, but this gets very complicated.
7292              * Without fixing here, we might actually need more checks.
7293              */
7294             if ((dd->comm->cell_x1[dim] - dd->comm->cell_x0[dim])*ddbox.skew_fac[dim]*dd->comm->cd[d].np_dlb < cutoff_req)
7295             {
7296                 LocallyLimited = 1;
7297             }
7298         }
7299     }
7300
7301     if (dd->comm->eDLB != edlbNO)
7302     {
7303         if (check_grid_jump(0,dd,cutoff_req,&ddbox,FALSE))
7304         {
7305             LocallyLimited = 1;
7306         }
7307
7308         gmx_sumi(1,&LocallyLimited,cr);
7309
7310         if (LocallyLimited > 0)
7311         {
7312             return FALSE;
7313         }
7314     }
7315
7316     dd->comm->cutoff = cutoff_req;
7317
7318     return TRUE;
7319 }
7320
7321 static void merge_cg_buffers(int ncell,
7322                              gmx_domdec_comm_dim_t *cd, int pulse,
7323                              int  *ncg_cell,
7324                              int  *index_gl, int  *recv_i,
7325                              rvec *cg_cm,    rvec *recv_vr,
7326                              int *cgindex,
7327                              cginfo_mb_t *cginfo_mb,int *cginfo)
7328 {
7329     gmx_domdec_ind_t *ind,*ind_p;
7330     int p,cell,c,cg,cg0,cg1,cg_gl,nat;
7331     int shift,shift_at;
7332
7333     ind = &cd->ind[pulse];
7334
7335     /* First correct the already stored data */
7336     shift = ind->nrecv[ncell];
7337     for(cell=ncell-1; cell>=0; cell--)
7338     {
7339         shift -= ind->nrecv[cell];
7340         if (shift > 0)
7341         {
7342             /* Move the cg's present from previous grid pulses */
7343             cg0 = ncg_cell[ncell+cell];
7344             cg1 = ncg_cell[ncell+cell+1];
7345             cgindex[cg1+shift] = cgindex[cg1];
7346             for(cg=cg1-1; cg>=cg0; cg--)
7347             {
7348                 index_gl[cg+shift] = index_gl[cg];
7349                 copy_rvec(cg_cm[cg],cg_cm[cg+shift]);
7350                 cgindex[cg+shift] = cgindex[cg];
7351                 cginfo[cg+shift] = cginfo[cg];
7352             }
7353             /* Correct the already stored send indices for the shift */
7354             for(p=1; p<=pulse; p++)
7355             {
7356                 ind_p = &cd->ind[p];
7357                 cg0 = 0;
7358                 for(c=0; c<cell; c++)
7359                 {
7360                     cg0 += ind_p->nsend[c];
7361                 }
7362                 cg1 = cg0 + ind_p->nsend[cell];
7363                 for(cg=cg0; cg<cg1; cg++)
7364                 {
7365                     ind_p->index[cg] += shift;
7366                 }
7367             }
7368         }
7369     }
7370
7371     /* Merge in the communicated buffers */
7372     shift = 0;
7373     shift_at = 0;
7374     cg0 = 0;
7375     for(cell=0; cell<ncell; cell++)
7376     {
7377         cg1 = ncg_cell[ncell+cell+1] + shift;
7378         if (shift_at > 0)
7379         {
7380             /* Correct the old cg indices */
7381             for(cg=ncg_cell[ncell+cell]; cg<cg1; cg++)
7382             {
7383                 cgindex[cg+1] += shift_at;
7384             }
7385         }
7386         for(cg=0; cg<ind->nrecv[cell]; cg++)
7387         {
7388             /* Copy this charge group from the buffer */
7389             index_gl[cg1] = recv_i[cg0];
7390             copy_rvec(recv_vr[cg0],cg_cm[cg1]);
7391             /* Add it to the cgindex */
7392             cg_gl = index_gl[cg1];
7393             cginfo[cg1] = ddcginfo(cginfo_mb,cg_gl);
7394             nat = GET_CGINFO_NATOMS(cginfo[cg1]);
7395             cgindex[cg1+1] = cgindex[cg1] + nat;
7396             cg0++;
7397             cg1++;
7398             shift_at += nat;
7399         }
7400         shift += ind->nrecv[cell];
7401         ncg_cell[ncell+cell+1] = cg1;
7402     }
7403 }
7404
7405 static void make_cell2at_index(gmx_domdec_comm_dim_t *cd,
7406                                int nzone,int cg0,const int *cgindex)
7407 {
7408     int cg,zone,p;
7409
7410     /* Store the atom block boundaries for easy copying of communication buffers
7411      */
7412     cg = cg0;
7413     for(zone=0; zone<nzone; zone++)
7414     {
7415         for(p=0; p<cd->np; p++) {
7416             cd->ind[p].cell2at0[zone] = cgindex[cg];
7417             cg += cd->ind[p].nrecv[zone];
7418             cd->ind[p].cell2at1[zone] = cgindex[cg];
7419         }
7420     }
7421 }
7422
7423 static gmx_bool missing_link(t_blocka *link,int cg_gl,char *bLocalCG)
7424 {
7425     int  i;
7426     gmx_bool bMiss;
7427
7428     bMiss = FALSE;
7429     for(i=link->index[cg_gl]; i<link->index[cg_gl+1]; i++)
7430     {
7431         if (!bLocalCG[link->a[i]])
7432         {
7433             bMiss = TRUE;
7434         }
7435     }
7436
7437     return bMiss;
7438 }
7439
7440 /* Domain corners for communication, a maximum of 4 i-zones see a j domain */
7441 typedef struct {
7442     real c[DIM][4]; /* the corners for the non-bonded communication */
7443     real cr0;       /* corner for rounding */
7444     real cr1[4];    /* corners for rounding */
7445     real bc[DIM];   /* corners for bounded communication */
7446     real bcr1;      /* corner for rounding for bonded communication */
7447 } dd_corners_t;
7448
7449 /* Determine the corners of the domain(s) we are communicating with */
7450 static void
7451 set_dd_corners(const gmx_domdec_t *dd,
7452                int dim0, int dim1, int dim2,
7453                gmx_bool bDistMB,
7454                dd_corners_t *c)
7455 {
7456     const gmx_domdec_comm_t *comm;
7457     const gmx_domdec_zones_t *zones;
7458     int i,j;
7459
7460     comm = dd->comm;
7461
7462     zones = &comm->zones;
7463
7464     /* Keep the compiler happy */
7465     c->cr0  = 0;
7466     c->bcr1 = 0;
7467
7468     /* The first dimension is equal for all cells */
7469     c->c[0][0] = comm->cell_x0[dim0];
7470     if (bDistMB)
7471     {
7472         c->bc[0] = c->c[0][0];
7473     }
7474     if (dd->ndim >= 2)
7475     {
7476         dim1 = dd->dim[1];
7477         /* This cell row is only seen from the first row */
7478         c->c[1][0] = comm->cell_x0[dim1];
7479         /* All rows can see this row */
7480         c->c[1][1] = comm->cell_x0[dim1];
7481         if (dd->bGridJump)
7482         {
7483             c->c[1][1] = max(comm->cell_x0[dim1],comm->zone_d1[1].mch0);
7484             if (bDistMB)
7485             {
7486                 /* For the multi-body distance we need the maximum */
7487                 c->bc[1] = max(comm->cell_x0[dim1],comm->zone_d1[1].p1_0);
7488             }
7489         }
7490         /* Set the upper-right corner for rounding */
7491         c->cr0 = comm->cell_x1[dim0];
7492
7493         if (dd->ndim >= 3)
7494         {
7495             dim2 = dd->dim[2];
7496             for(j=0; j<4; j++)
7497             {
7498                 c->c[2][j] = comm->cell_x0[dim2];
7499             }
7500             if (dd->bGridJump)
7501             {
7502                 /* Use the maximum of the i-cells that see a j-cell */
7503                 for(i=0; i<zones->nizone; i++)
7504                 {
7505                     for(j=zones->izone[i].j0; j<zones->izone[i].j1; j++)
7506                     {
7507                         if (j >= 4)
7508                         {
7509                             c->c[2][j-4] =
7510                                 max(c->c[2][j-4],
7511                                     comm->zone_d2[zones->shift[i][dim0]][zones->shift[i][dim1]].mch0);
7512                         }
7513                     }
7514                 }
7515                 if (bDistMB)
7516                 {
7517                     /* For the multi-body distance we need the maximum */
7518                     c->bc[2] = comm->cell_x0[dim2];
7519                     for(i=0; i<2; i++)
7520                     {
7521                         for(j=0; j<2; j++)
7522                         {
7523                             c->bc[2] = max(c->bc[2],comm->zone_d2[i][j].p1_0);
7524                         }
7525                     }
7526                 }
7527             }
7528
7529             /* Set the upper-right corner for rounding */
7530             /* Cell (0,0,0) and cell (1,0,0) can see cell 4 (0,1,1)
7531              * Only cell (0,0,0) can see cell 7 (1,1,1)
7532              */
7533             c->cr1[0] = comm->cell_x1[dim1];
7534             c->cr1[3] = comm->cell_x1[dim1];
7535             if (dd->bGridJump)
7536             {
7537                 c->cr1[0] = max(comm->cell_x1[dim1],comm->zone_d1[1].mch1);
7538                 if (bDistMB)
7539                 {
7540                     /* For the multi-body distance we need the maximum */
7541                     c->bcr1 = max(comm->cell_x1[dim1],comm->zone_d1[1].p1_1);
7542                 }
7543             }
7544         }
7545     }
7546 }
7547
7548 /* Determine which cg's we need to send in this pulse from this zone */
7549 static void
7550 get_zone_pulse_cgs(gmx_domdec_t *dd,
7551                    int zonei, int zone,
7552                    int cg0, int cg1,
7553                    const int *index_gl,
7554                    const int *cgindex,
7555                    int dim, int dim_ind,
7556                    int dim0, int dim1, int dim2,
7557                    real r_comm2, real r_bcomm2,
7558                    matrix box,
7559                    ivec tric_dist,
7560                    rvec *normal,
7561                    real skew_fac2_d, real skew_fac_01,
7562                    rvec *v_d, rvec *v_0, rvec *v_1,
7563                    const dd_corners_t *c,
7564                    rvec sf2_round,
7565                    gmx_bool bDistBonded,
7566                    gmx_bool bBondComm,
7567                    gmx_bool bDist2B,
7568                    gmx_bool bDistMB,
7569                    rvec *cg_cm,
7570                    int *cginfo,
7571                    gmx_domdec_ind_t *ind,
7572                    int **ibuf, int *ibuf_nalloc,
7573                    vec_rvec_t *vbuf,
7574                    int *nsend_ptr,
7575                    int *nat_ptr,
7576                    int *nsend_z_ptr)
7577 {
7578     gmx_domdec_comm_t *comm;
7579     gmx_bool bScrew;
7580     gmx_bool bDistMB_pulse;
7581     int  cg,i;
7582     real r2,rb2,r,tric_sh;
7583     rvec rn,rb;
7584     int  dimd;
7585     int  nsend_z,nsend,nat;
7586
7587     comm = dd->comm;
7588
7589     bScrew = (dd->bScrewPBC && dim == XX);
7590
7591     bDistMB_pulse = (bDistMB && bDistBonded);
7592
7593     nsend_z = 0;
7594     nsend   = *nsend_ptr;
7595     nat     = *nat_ptr;
7596
7597     for(cg=cg0; cg<cg1; cg++)
7598     {
7599         r2  = 0;
7600         rb2 = 0;
7601         if (tric_dist[dim_ind] == 0)
7602         {
7603             /* Rectangular direction, easy */
7604             r = cg_cm[cg][dim] - c->c[dim_ind][zone];
7605             if (r > 0)
7606             {
7607                 r2 += r*r;
7608             }
7609             if (bDistMB_pulse)
7610             {
7611                 r = cg_cm[cg][dim] - c->bc[dim_ind];
7612                 if (r > 0)
7613                 {
7614                     rb2 += r*r;
7615                 }
7616             }
7617             /* Rounding gives at most a 16% reduction
7618              * in communicated atoms
7619              */
7620             if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
7621             {
7622                 r = cg_cm[cg][dim0] - c->cr0;
7623                 /* This is the first dimension, so always r >= 0 */
7624                 r2 += r*r;
7625                 if (bDistMB_pulse)
7626                 {
7627                     rb2 += r*r;
7628                 }
7629             }
7630             if (dim_ind == 2 && (zonei == 2 || zonei == 3))
7631             {
7632                 r = cg_cm[cg][dim1] - c->cr1[zone];
7633                 if (r > 0)
7634                 {
7635                     r2 += r*r;
7636                 }
7637                 if (bDistMB_pulse)
7638                 {
7639                     r = cg_cm[cg][dim1] - c->bcr1;
7640                     if (r > 0)
7641                     {
7642                         rb2 += r*r;
7643                     }
7644                 }
7645             }
7646         }
7647         else
7648         {
7649             /* Triclinic direction, more complicated */
7650             clear_rvec(rn);
7651             clear_rvec(rb);
7652             /* Rounding, conservative as the skew_fac multiplication
7653              * will slightly underestimate the distance.
7654              */
7655             if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
7656             {
7657                 rn[dim0] = cg_cm[cg][dim0] - c->cr0;
7658                 for(i=dim0+1; i<DIM; i++)
7659                 {
7660                     rn[dim0] -= cg_cm[cg][i]*v_0[i][dim0];
7661                 }
7662                 r2 = rn[dim0]*rn[dim0]*sf2_round[dim0];
7663                 if (bDistMB_pulse)
7664                 {
7665                     rb[dim0] = rn[dim0];
7666                     rb2 = r2;
7667                 }
7668                 /* Take care that the cell planes along dim0 might not
7669                  * be orthogonal to those along dim1 and dim2.
7670                  */
7671                 for(i=1; i<=dim_ind; i++)
7672                 {
7673                     dimd = dd->dim[i];
7674                     if (normal[dim0][dimd] > 0)
7675                     {
7676                         rn[dimd] -= rn[dim0]*normal[dim0][dimd];
7677                         if (bDistMB_pulse)
7678                         {
7679                             rb[dimd] -= rb[dim0]*normal[dim0][dimd];
7680                         }
7681                     }
7682                 }
7683             }
7684             if (dim_ind == 2 && (zonei == 2 || zonei == 3))
7685             {
7686                 rn[dim1] += cg_cm[cg][dim1] - c->cr1[zone];
7687                 tric_sh = 0;
7688                 for(i=dim1+1; i<DIM; i++)
7689                 {
7690                     tric_sh -= cg_cm[cg][i]*v_1[i][dim1];
7691                 }
7692                 rn[dim1] += tric_sh;
7693                 if (rn[dim1] > 0)
7694                 {
7695                     r2 += rn[dim1]*rn[dim1]*sf2_round[dim1];
7696                     /* Take care of coupling of the distances
7697                      * to the planes along dim0 and dim1 through dim2.
7698                      */
7699                     r2 -= rn[dim0]*rn[dim1]*skew_fac_01;
7700                     /* Take care that the cell planes along dim1
7701                      * might not be orthogonal to that along dim2.
7702                      */
7703                     if (normal[dim1][dim2] > 0)
7704                     {
7705                         rn[dim2] -= rn[dim1]*normal[dim1][dim2];
7706                     }
7707                 }
7708                 if (bDistMB_pulse)
7709                 {
7710                     rb[dim1] +=
7711                         cg_cm[cg][dim1] - c->bcr1 + tric_sh;
7712                     if (rb[dim1] > 0)
7713                     {
7714                         rb2 += rb[dim1]*rb[dim1]*sf2_round[dim1];
7715                         /* Take care of coupling of the distances
7716                          * to the planes along dim0 and dim1 through dim2.
7717                          */
7718                         rb2 -= rb[dim0]*rb[dim1]*skew_fac_01;
7719                         /* Take care that the cell planes along dim1
7720                          * might not be orthogonal to that along dim2.
7721                          */
7722                         if (normal[dim1][dim2] > 0)
7723                         {
7724                             rb[dim2] -= rb[dim1]*normal[dim1][dim2];
7725                         }
7726                     }
7727                 }
7728             }
7729             /* The distance along the communication direction */
7730             rn[dim] += cg_cm[cg][dim] - c->c[dim_ind][zone];
7731             tric_sh = 0;
7732             for(i=dim+1; i<DIM; i++)
7733             {
7734                 tric_sh -= cg_cm[cg][i]*v_d[i][dim];
7735             }
7736             rn[dim] += tric_sh;
7737             if (rn[dim] > 0)
7738             {
7739                 r2 += rn[dim]*rn[dim]*skew_fac2_d;
7740                 /* Take care of coupling of the distances
7741                  * to the planes along dim0 and dim1 through dim2.
7742                  */
7743                 if (dim_ind == 1 && zonei == 1)
7744                 {
7745                     r2 -= rn[dim0]*rn[dim]*skew_fac_01;
7746                 }
7747             }
7748             if (bDistMB_pulse)
7749             {
7750                 clear_rvec(rb);
7751                 rb[dim] += cg_cm[cg][dim] - c->bc[dim_ind] + tric_sh;
7752                 if (rb[dim] > 0)
7753                 {
7754                     rb2 += rb[dim]*rb[dim]*skew_fac2_d;
7755                     /* Take care of coupling of the distances
7756                      * to the planes along dim0 and dim1 through dim2.
7757                      */
7758                     if (dim_ind == 1 && zonei == 1)
7759                     {
7760                         rb2 -= rb[dim0]*rb[dim]*skew_fac_01;
7761                     }
7762                 }
7763             }
7764         }
7765
7766         if (r2 < r_comm2 ||
7767             (bDistBonded &&
7768              ((bDistMB && rb2 < r_bcomm2) ||
7769               (bDist2B && r2  < r_bcomm2)) &&
7770              (!bBondComm ||
7771               (GET_CGINFO_BOND_INTER(cginfo[cg]) &&
7772                missing_link(comm->cglink,index_gl[cg],
7773                             comm->bLocalCG)))))
7774         {
7775             /* Make an index to the local charge groups */
7776             if (nsend+1 > ind->nalloc)
7777             {
7778                 ind->nalloc = over_alloc_large(nsend+1);
7779                 srenew(ind->index,ind->nalloc);
7780             }
7781             if (nsend+1 > *ibuf_nalloc)
7782             {
7783                 *ibuf_nalloc = over_alloc_large(nsend+1);
7784                 srenew(*ibuf,*ibuf_nalloc);
7785             }
7786             ind->index[nsend] = cg;
7787             (*ibuf)[nsend] = index_gl[cg];
7788             nsend_z++;
7789             vec_rvec_check_alloc(vbuf,nsend+1);
7790
7791             if (dd->ci[dim] == 0)
7792             {
7793                 /* Correct cg_cm for pbc */
7794                 rvec_add(cg_cm[cg],box[dim],vbuf->v[nsend]);
7795                 if (bScrew)
7796                 {
7797                     vbuf->v[nsend][YY] = box[YY][YY] - vbuf->v[nsend][YY];
7798                     vbuf->v[nsend][ZZ] = box[ZZ][ZZ] - vbuf->v[nsend][ZZ];
7799                 }
7800             }
7801             else
7802             {
7803                 copy_rvec(cg_cm[cg],vbuf->v[nsend]);
7804             }
7805             nsend++;
7806             nat += cgindex[cg+1] - cgindex[cg];
7807         }
7808     }
7809
7810     *nsend_ptr   = nsend;
7811     *nat_ptr     = nat;
7812     *nsend_z_ptr = nsend_z;
7813 }
7814
7815 static void setup_dd_communication(gmx_domdec_t *dd,
7816                                    matrix box,gmx_ddbox_t *ddbox,
7817                                    t_forcerec *fr,t_state *state,rvec **f)
7818 {
7819     int dim_ind,dim,dim0,dim1,dim2,dimd,p,nat_tot;
7820     int nzone,nzone_send,zone,zonei,cg0,cg1;
7821     int c,i,j,cg,cg_gl,nrcg;
7822     int *zone_cg_range,pos_cg,*index_gl,*cgindex,*recv_i;
7823     gmx_domdec_comm_t *comm;
7824     gmx_domdec_zones_t *zones;
7825     gmx_domdec_comm_dim_t *cd;
7826     gmx_domdec_ind_t *ind;
7827     cginfo_mb_t *cginfo_mb;
7828     gmx_bool bBondComm,bDist2B,bDistMB,bDistBonded;
7829     real r_mb,r_comm2,r_scomm2,r_bcomm2,r_0,r_1,r2inc,inv_ncg;
7830     dd_corners_t corners;
7831     ivec tric_dist;
7832     rvec *cg_cm,*normal,*v_d,*v_0=NULL,*v_1=NULL,*recv_vr;
7833     real skew_fac2_d,skew_fac_01;
7834     rvec sf2_round;
7835     int  nsend,nat;
7836     int  th;
7837
7838     if (debug)
7839     {
7840         fprintf(debug,"Setting up DD communication\n");
7841     }
7842
7843     comm  = dd->comm;
7844
7845     switch (fr->cutoff_scheme)
7846     {
7847     case ecutsGROUP:
7848         cg_cm = fr->cg_cm;
7849         break;
7850     case ecutsVERLET:
7851         cg_cm = state->x;
7852         break;
7853     default:
7854         gmx_incons("unimplemented");
7855         cg_cm = NULL;
7856     }
7857
7858     for(dim_ind=0; dim_ind<dd->ndim; dim_ind++)
7859     {
7860         dim = dd->dim[dim_ind];
7861
7862         /* Check if we need to use triclinic distances */
7863         tric_dist[dim_ind] = 0;
7864         for(i=0; i<=dim_ind; i++)
7865         {
7866             if (ddbox->tric_dir[dd->dim[i]])
7867             {
7868                 tric_dist[dim_ind] = 1;
7869             }
7870         }
7871     }
7872
7873     bBondComm = comm->bBondComm;
7874
7875     /* Do we need to determine extra distances for multi-body bondeds? */
7876     bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
7877
7878     /* Do we need to determine extra distances for only two-body bondeds? */
7879     bDist2B = (bBondComm && !bDistMB);
7880
7881     r_comm2  = sqr(comm->cutoff);
7882     r_bcomm2 = sqr(comm->cutoff_mbody);
7883
7884     if (debug)
7885     {
7886         fprintf(debug,"bBondComm %d, r_bc %f\n",bBondComm,sqrt(r_bcomm2));
7887     }
7888
7889     zones = &comm->zones;
7890
7891     dim0 = dd->dim[0];
7892     dim1 = (dd->ndim >= 2 ? dd->dim[1] : -1);
7893     dim2 = (dd->ndim >= 3 ? dd->dim[2] : -1);
7894
7895     set_dd_corners(dd,dim0,dim1,dim2,bDistMB,&corners);
7896
7897     /* Triclinic stuff */
7898     normal = ddbox->normal;
7899     skew_fac_01 = 0;
7900     if (dd->ndim >= 2)
7901     {
7902         v_0 = ddbox->v[dim0];
7903         if (ddbox->tric_dir[dim0] && ddbox->tric_dir[dim1])
7904         {
7905             /* Determine the coupling coefficient for the distances
7906              * to the cell planes along dim0 and dim1 through dim2.
7907              * This is required for correct rounding.
7908              */
7909             skew_fac_01 =
7910                 ddbox->v[dim0][dim1+1][dim0]*ddbox->v[dim1][dim1+1][dim1];
7911             if (debug)
7912             {
7913                 fprintf(debug,"\nskew_fac_01 %f\n",skew_fac_01);
7914             }
7915         }
7916     }
7917     if (dd->ndim >= 3)
7918     {
7919         v_1 = ddbox->v[dim1];
7920     }
7921
7922     zone_cg_range = zones->cg_range;
7923     index_gl = dd->index_gl;
7924     cgindex  = dd->cgindex;
7925     cginfo_mb = fr->cginfo_mb;
7926
7927     zone_cg_range[0]   = 0;
7928     zone_cg_range[1]   = dd->ncg_home;
7929     comm->zone_ncg1[0] = dd->ncg_home;
7930     pos_cg             = dd->ncg_home;
7931
7932     nat_tot = dd->nat_home;
7933     nzone = 1;
7934     for(dim_ind=0; dim_ind<dd->ndim; dim_ind++)
7935     {
7936         dim = dd->dim[dim_ind];
7937         cd = &comm->cd[dim_ind];
7938
7939         if (dim >= ddbox->npbcdim && dd->ci[dim] == 0)
7940         {
7941             /* No pbc in this dimension, the first node should not comm. */
7942             nzone_send = 0;
7943         }
7944         else
7945         {
7946             nzone_send = nzone;
7947         }
7948
7949         v_d = ddbox->v[dim];
7950         skew_fac2_d = sqr(ddbox->skew_fac[dim]);
7951
7952         cd->bInPlace = TRUE;
7953         for(p=0; p<cd->np; p++)
7954         {
7955             /* Only atoms communicated in the first pulse are used
7956              * for multi-body bonded interactions or for bBondComm.
7957              */
7958             bDistBonded = ((bDistMB || bDist2B) && p == 0);
7959
7960             ind = &cd->ind[p];
7961             nsend = 0;
7962             nat = 0;
7963             for(zone=0; zone<nzone_send; zone++)
7964             {
7965                 if (tric_dist[dim_ind] && dim_ind > 0)
7966                 {
7967                     /* Determine slightly more optimized skew_fac's
7968                      * for rounding.
7969                      * This reduces the number of communicated atoms
7970                      * by about 10% for 3D DD of rhombic dodecahedra.
7971                      */
7972                     for(dimd=0; dimd<dim; dimd++)
7973                     {
7974                         sf2_round[dimd] = 1;
7975                         if (ddbox->tric_dir[dimd])
7976                         {
7977                             for(i=dd->dim[dimd]+1; i<DIM; i++)
7978                             {
7979                                 /* If we are shifted in dimension i
7980                                  * and the cell plane is tilted forward
7981                                  * in dimension i, skip this coupling.
7982                                  */
7983                                 if (!(zones->shift[nzone+zone][i] &&
7984                                       ddbox->v[dimd][i][dimd] >= 0))
7985                                 {
7986                                     sf2_round[dimd] +=
7987                                         sqr(ddbox->v[dimd][i][dimd]);
7988                                 }
7989                             }
7990                             sf2_round[dimd] = 1/sf2_round[dimd];
7991                         }
7992                     }
7993                 }
7994
7995                 zonei = zone_perm[dim_ind][zone];
7996                 if (p == 0)
7997                 {
7998                     /* Here we permutate the zones to obtain a convenient order
7999                      * for neighbor searching
8000                      */
8001                     cg0 = zone_cg_range[zonei];
8002                     cg1 = zone_cg_range[zonei+1];
8003                 }
8004                 else
8005                 {
8006                     /* Look only at the cg's received in the previous grid pulse
8007                      */
8008                     cg1 = zone_cg_range[nzone+zone+1];
8009                     cg0 = cg1 - cd->ind[p-1].nrecv[zone];
8010                 }
8011
8012 #pragma omp parallel for num_threads(comm->nth) schedule(static)
8013                 for(th=0; th<comm->nth; th++)
8014                 {
8015                     gmx_domdec_ind_t *ind_p;
8016                     int **ibuf_p,*ibuf_nalloc_p;
8017                     vec_rvec_t *vbuf_p;
8018                     int *nsend_p,*nat_p;
8019                     int *nsend_zone_p;
8020                     int cg0_th,cg1_th;
8021
8022                     if (th == 0)
8023                     {
8024                         /* Thread 0 writes in the comm buffers */
8025                         ind_p         = ind;
8026                         ibuf_p        = &comm->buf_int;
8027                         ibuf_nalloc_p = &comm->nalloc_int;
8028                         vbuf_p        = &comm->vbuf;
8029                         nsend_p       = &nsend;
8030                         nat_p         = &nat;
8031                         nsend_zone_p  = &ind->nsend[zone];
8032                     }
8033                     else
8034                     {
8035                         /* Other threads write into temp buffers */
8036                         ind_p         = &comm->dth[th].ind;
8037                         ibuf_p        = &comm->dth[th].ibuf;
8038                         ibuf_nalloc_p = &comm->dth[th].ibuf_nalloc;
8039                         vbuf_p        = &comm->dth[th].vbuf;
8040                         nsend_p       = &comm->dth[th].nsend;
8041                         nat_p         = &comm->dth[th].nat;
8042                         nsend_zone_p  = &comm->dth[th].nsend_zone;
8043
8044                         comm->dth[th].nsend      = 0;
8045                         comm->dth[th].nat        = 0;
8046                         comm->dth[th].nsend_zone = 0;
8047                     }
8048
8049                     if (comm->nth == 1)
8050                     {
8051                         cg0_th = cg0;
8052                         cg1_th = cg1;
8053                     }
8054                     else
8055                     {
8056                         cg0_th = cg0 + ((cg1 - cg0)* th   )/comm->nth;
8057                         cg1_th = cg0 + ((cg1 - cg0)*(th+1))/comm->nth;
8058                     }
8059
8060                     /* Get the cg's for this pulse in this zone */
8061                     get_zone_pulse_cgs(dd,zonei,zone,cg0_th,cg1_th,
8062                                        index_gl,cgindex,
8063                                        dim,dim_ind,dim0,dim1,dim2,
8064                                        r_comm2,r_bcomm2,
8065                                        box,tric_dist,
8066                                        normal,skew_fac2_d,skew_fac_01,
8067                                        v_d,v_0,v_1,&corners,sf2_round,
8068                                        bDistBonded,bBondComm,
8069                                        bDist2B,bDistMB,
8070                                        cg_cm,fr->cginfo,
8071                                        ind_p,
8072                                        ibuf_p,ibuf_nalloc_p,
8073                                        vbuf_p,
8074                                        nsend_p,nat_p,
8075                                        nsend_zone_p);
8076                 }
8077
8078                 /* Append data of threads>=1 to the communication buffers */
8079                 for(th=1; th<comm->nth; th++)
8080                 {
8081                     dd_comm_setup_work_t *dth;
8082                     int i,ns1;
8083
8084                     dth = &comm->dth[th];
8085
8086                     ns1 = nsend + dth->nsend_zone;
8087                     if (ns1 > ind->nalloc)
8088                     {
8089                         ind->nalloc = over_alloc_dd(ns1);
8090                         srenew(ind->index,ind->nalloc);
8091                     }
8092                     if (ns1 > comm->nalloc_int)
8093                     {
8094                         comm->nalloc_int = over_alloc_dd(ns1);
8095                         srenew(comm->buf_int,comm->nalloc_int);
8096                     }
8097                     if (ns1 > comm->vbuf.nalloc)
8098                     {
8099                         comm->vbuf.nalloc = over_alloc_dd(ns1);
8100                         srenew(comm->vbuf.v,comm->vbuf.nalloc);
8101                     }
8102
8103                     for(i=0; i<dth->nsend_zone; i++)
8104                     {
8105                         ind->index[nsend] = dth->ind.index[i];
8106                         comm->buf_int[nsend] = dth->ibuf[i];
8107                         copy_rvec(dth->vbuf.v[i],
8108                                   comm->vbuf.v[nsend]);
8109                         nsend++;
8110                     }
8111                     nat              += dth->nat;
8112                     ind->nsend[zone] += dth->nsend_zone;
8113                 }
8114             }
8115             /* Clear the counts in case we do not have pbc */
8116             for(zone=nzone_send; zone<nzone; zone++)
8117             {
8118                 ind->nsend[zone] = 0;
8119             }
8120             ind->nsend[nzone]   = nsend;
8121             ind->nsend[nzone+1] = nat;
8122             /* Communicate the number of cg's and atoms to receive */
8123             dd_sendrecv_int(dd, dim_ind, dddirBackward,
8124                             ind->nsend, nzone+2,
8125                             ind->nrecv, nzone+2);
8126
8127             /* The rvec buffer is also required for atom buffers of size nsend
8128              * in dd_move_x and dd_move_f.
8129              */
8130             vec_rvec_check_alloc(&comm->vbuf,ind->nsend[nzone+1]);
8131
8132             if (p > 0)
8133             {
8134                 /* We can receive in place if only the last zone is not empty */
8135                 for(zone=0; zone<nzone-1; zone++)
8136                 {
8137                     if (ind->nrecv[zone] > 0)
8138                     {
8139                         cd->bInPlace = FALSE;
8140                     }
8141                 }
8142                 if (!cd->bInPlace)
8143                 {
8144                     /* The int buffer is only required here for the cg indices */
8145                     if (ind->nrecv[nzone] > comm->nalloc_int2)
8146                     {
8147                         comm->nalloc_int2 = over_alloc_dd(ind->nrecv[nzone]);
8148                         srenew(comm->buf_int2,comm->nalloc_int2);
8149                     }
8150                     /* The rvec buffer is also required for atom buffers
8151                      * of size nrecv in dd_move_x and dd_move_f.
8152                      */
8153                     i = max(cd->ind[0].nrecv[nzone+1],ind->nrecv[nzone+1]);
8154                     vec_rvec_check_alloc(&comm->vbuf2,i);
8155                 }
8156             }
8157
8158             /* Make space for the global cg indices */
8159             if (pos_cg + ind->nrecv[nzone] > dd->cg_nalloc
8160                 || dd->cg_nalloc == 0)
8161             {
8162                 dd->cg_nalloc = over_alloc_dd(pos_cg + ind->nrecv[nzone]);
8163                 srenew(index_gl,dd->cg_nalloc);
8164                 srenew(cgindex,dd->cg_nalloc+1);
8165             }
8166             /* Communicate the global cg indices */
8167             if (cd->bInPlace)
8168             {
8169                 recv_i = index_gl + pos_cg;
8170             }
8171             else
8172             {
8173                 recv_i = comm->buf_int2;
8174             }
8175             dd_sendrecv_int(dd, dim_ind, dddirBackward,
8176                             comm->buf_int, nsend,
8177                             recv_i,        ind->nrecv[nzone]);
8178
8179             /* Make space for cg_cm */
8180             dd_check_alloc_ncg(fr,state,f,pos_cg + ind->nrecv[nzone]);
8181             if (fr->cutoff_scheme == ecutsGROUP)
8182             {
8183                 cg_cm = fr->cg_cm;
8184             }
8185             else
8186             {
8187                 cg_cm = state->x;
8188             }
8189             /* Communicate cg_cm */
8190             if (cd->bInPlace)
8191             {
8192                 recv_vr = cg_cm + pos_cg;
8193             }
8194             else
8195             {
8196                 recv_vr = comm->vbuf2.v;
8197             }
8198             dd_sendrecv_rvec(dd, dim_ind, dddirBackward,
8199                              comm->vbuf.v, nsend,
8200                              recv_vr,      ind->nrecv[nzone]);
8201
8202             /* Make the charge group index */
8203             if (cd->bInPlace)
8204             {
8205                 zone = (p == 0 ? 0 : nzone - 1);
8206                 while (zone < nzone)
8207                 {
8208                     for(cg=0; cg<ind->nrecv[zone]; cg++)
8209                     {
8210                         cg_gl = index_gl[pos_cg];
8211                         fr->cginfo[pos_cg] = ddcginfo(cginfo_mb,cg_gl);
8212                         nrcg = GET_CGINFO_NATOMS(fr->cginfo[pos_cg]);
8213                         cgindex[pos_cg+1] = cgindex[pos_cg] + nrcg;
8214                         if (bBondComm)
8215                         {
8216                             /* Update the charge group presence,
8217                              * so we can use it in the next pass of the loop.
8218                              */
8219                             comm->bLocalCG[cg_gl] = TRUE;
8220                         }
8221                         pos_cg++;
8222                     }
8223                     if (p == 0)
8224                     {
8225                         comm->zone_ncg1[nzone+zone] = ind->nrecv[zone];
8226                     }
8227                     zone++;
8228                     zone_cg_range[nzone+zone] = pos_cg;
8229                 }
8230             }
8231             else
8232             {
8233                 /* This part of the code is never executed with bBondComm. */
8234                 merge_cg_buffers(nzone,cd,p,zone_cg_range,
8235                                  index_gl,recv_i,cg_cm,recv_vr,
8236                                  cgindex,fr->cginfo_mb,fr->cginfo);
8237                 pos_cg += ind->nrecv[nzone];
8238             }
8239             nat_tot += ind->nrecv[nzone+1];
8240         }
8241         if (!cd->bInPlace)
8242         {
8243             /* Store the atom block for easy copying of communication buffers */
8244             make_cell2at_index(cd,nzone,zone_cg_range[nzone],cgindex);
8245         }
8246         nzone += nzone;
8247     }
8248     dd->index_gl = index_gl;
8249     dd->cgindex  = cgindex;
8250
8251     dd->ncg_tot = zone_cg_range[zones->n];
8252     dd->nat_tot = nat_tot;
8253     comm->nat[ddnatHOME] = dd->nat_home;
8254     for(i=ddnatZONE; i<ddnatNR; i++)
8255     {
8256         comm->nat[i] = dd->nat_tot;
8257     }
8258
8259     if (!bBondComm)
8260     {
8261         /* We don't need to update cginfo, since that was alrady done above.
8262          * So we pass NULL for the forcerec.
8263          */
8264         dd_set_cginfo(dd->index_gl,dd->ncg_home,dd->ncg_tot,
8265                       NULL,comm->bLocalCG);
8266     }
8267
8268     if (debug)
8269     {
8270         fprintf(debug,"Finished setting up DD communication, zones:");
8271         for(c=0; c<zones->n; c++)
8272         {
8273             fprintf(debug," %d",zones->cg_range[c+1]-zones->cg_range[c]);
8274         }
8275         fprintf(debug,"\n");
8276     }
8277 }
8278
8279 static void set_cg_boundaries(gmx_domdec_zones_t *zones)
8280 {
8281     int c;
8282
8283     for(c=0; c<zones->nizone; c++)
8284     {
8285         zones->izone[c].cg1  = zones->cg_range[c+1];
8286         zones->izone[c].jcg0 = zones->cg_range[zones->izone[c].j0];
8287         zones->izone[c].jcg1 = zones->cg_range[zones->izone[c].j1];
8288     }
8289 }
8290
8291 static void set_zones_size(gmx_domdec_t *dd,
8292                            matrix box,const gmx_ddbox_t *ddbox,
8293                            int zone_start,int zone_end)
8294 {
8295     gmx_domdec_comm_t *comm;
8296     gmx_domdec_zones_t *zones;
8297     gmx_bool bDistMB;
8298     int  z,zi,zj0,zj1,d,dim;
8299     real rcs,rcmbs;
8300     int  i,j;
8301     real size_j,add_tric;
8302     real vol;
8303
8304     comm = dd->comm;
8305
8306     zones = &comm->zones;
8307
8308     /* Do we need to determine extra distances for multi-body bondeds? */
8309     bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
8310
8311     for(z=zone_start; z<zone_end; z++)
8312     {
8313         /* Copy cell limits to zone limits.
8314          * Valid for non-DD dims and non-shifted dims.
8315          */
8316         copy_rvec(comm->cell_x0,zones->size[z].x0);
8317         copy_rvec(comm->cell_x1,zones->size[z].x1);
8318     }
8319
8320     for(d=0; d<dd->ndim; d++)
8321     {
8322         dim = dd->dim[d];
8323
8324         for(z=0; z<zones->n; z++)
8325         {
8326             /* With a staggered grid we have different sizes
8327              * for non-shifted dimensions.
8328              */
8329             if (dd->bGridJump && zones->shift[z][dim] == 0)
8330             {
8331                 if (d == 1)
8332                 {
8333                     zones->size[z].x0[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min0;
8334                     zones->size[z].x1[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].max1;
8335                 }
8336                 else if (d == 2)
8337                 {
8338                     zones->size[z].x0[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min0;
8339                     zones->size[z].x1[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].max1;
8340                 }
8341             }
8342         }
8343
8344         rcs   = comm->cutoff;
8345         rcmbs = comm->cutoff_mbody;
8346         if (ddbox->tric_dir[dim])
8347         {
8348             rcs   /= ddbox->skew_fac[dim];
8349             rcmbs /= ddbox->skew_fac[dim];
8350         }
8351
8352         /* Set the lower limit for the shifted zone dimensions */
8353         for(z=zone_start; z<zone_end; z++)
8354         {
8355             if (zones->shift[z][dim] > 0)
8356             {
8357                 dim = dd->dim[d];
8358                 if (!dd->bGridJump || d == 0)
8359                 {
8360                     zones->size[z].x0[dim] = comm->cell_x1[dim];
8361                     zones->size[z].x1[dim] = comm->cell_x1[dim] + rcs;
8362                 }
8363                 else
8364                 {
8365                     /* Here we take the lower limit of the zone from
8366                      * the lowest domain of the zone below.
8367                      */
8368                     if (z < 4)
8369                     {
8370                         zones->size[z].x0[dim] =
8371                              comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min1;
8372                     }
8373                     else
8374                     {
8375                         if (d == 1)
8376                         {
8377                             zones->size[z].x0[dim] =
8378                                 zones->size[zone_perm[2][z-4]].x0[dim];
8379                         }
8380                         else
8381                         {
8382                             zones->size[z].x0[dim] =
8383                                 comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min1;
8384                         }
8385                     }
8386                     /* A temporary limit, is updated below */
8387                     zones->size[z].x1[dim] = zones->size[z].x0[dim];
8388
8389                     if (bDistMB)
8390                     {
8391                         for(zi=0; zi<zones->nizone; zi++)
8392                         {
8393                             if (zones->shift[zi][dim] == 0)
8394                             {
8395                                 /* This takes the whole zone into account.
8396                                  * With multiple pulses this will lead
8397                                  * to a larger zone then strictly necessary.
8398                                  */
8399                                 zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
8400                                                              zones->size[zi].x1[dim]+rcmbs);
8401                             }
8402                         }
8403                     }
8404                 }
8405             }
8406         }
8407
8408         /* Loop over the i-zones to set the upper limit of each
8409          * j-zone they see.
8410          */
8411         for(zi=0; zi<zones->nizone; zi++)
8412         {
8413             if (zones->shift[zi][dim] == 0)
8414             {
8415                 for(z=zones->izone[zi].j0; z<zones->izone[zi].j1; z++)
8416                 {
8417                     if (zones->shift[z][dim] > 0)
8418                     {
8419                         zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
8420                                                      zones->size[zi].x1[dim]+rcs);
8421                     }
8422                 }
8423             }
8424         }
8425     }
8426
8427     for(z=zone_start; z<zone_end; z++)
8428     {
8429         for(i=0; i<DIM; i++)
8430         {
8431             zones->size[z].bb_x0[i] = zones->size[z].x0[i];
8432             zones->size[z].bb_x1[i] = zones->size[z].x1[i];
8433
8434             for(j=i+1; j<ddbox->npbcdim; j++)
8435             {
8436                 /* With 1D domain decomposition the cg's are not in
8437                  * the triclinic box, but trilinic x-y and rectangular y-z.
8438                  */
8439                 if (box[j][i] != 0 &&
8440                     !(dd->ndim == 1 && i == YY && j == ZZ))
8441                 {
8442                     /* Correct for triclinic offset of the lower corner */
8443                     add_tric = zones->size[z].x0[j]*box[j][i]/box[j][j];
8444                     zones->size[z].bb_x0[i] += add_tric;
8445                     zones->size[z].bb_x1[i] += add_tric;
8446
8447                     /* Correct for triclinic offset of the upper corner */
8448                     size_j = zones->size[z].x1[j] - zones->size[z].x0[j];
8449                     add_tric = size_j*box[j][i]/box[j][j];
8450
8451                     if (box[j][i] < 0)
8452                     {
8453                         zones->size[z].bb_x0[i] += add_tric;
8454                     }
8455                     else
8456                     {
8457                         zones->size[z].bb_x1[i] += add_tric;
8458                     }
8459                 }
8460             }
8461         }
8462     }
8463
8464     if (zone_start == 0)
8465     {
8466         vol = 1;
8467         for(dim=0; dim<DIM; dim++)
8468         {
8469             vol *= zones->size[0].x1[dim] - zones->size[0].x0[dim];
8470         }
8471         zones->dens_zone0 = (zones->cg_range[1] - zones->cg_range[0])/vol;
8472     }
8473
8474     if (debug)
8475     {
8476         for(z=zone_start; z<zone_end; z++)
8477         {
8478             fprintf(debug,"zone %d    %6.3f - %6.3f  %6.3f - %6.3f  %6.3f - %6.3f\n",
8479                     z,
8480                     zones->size[z].x0[XX],zones->size[z].x1[XX],
8481                     zones->size[z].x0[YY],zones->size[z].x1[YY],
8482                     zones->size[z].x0[ZZ],zones->size[z].x1[ZZ]);
8483             fprintf(debug,"zone %d bb %6.3f - %6.3f  %6.3f - %6.3f  %6.3f - %6.3f\n",
8484                     z,
8485                     zones->size[z].bb_x0[XX],zones->size[z].bb_x1[XX],
8486                     zones->size[z].bb_x0[YY],zones->size[z].bb_x1[YY],
8487                     zones->size[z].bb_x0[ZZ],zones->size[z].bb_x1[ZZ]);
8488         }
8489     }
8490 }
8491
8492 static int comp_cgsort(const void *a,const void *b)
8493 {
8494     int comp;
8495
8496     gmx_cgsort_t *cga,*cgb;
8497     cga = (gmx_cgsort_t *)a;
8498     cgb = (gmx_cgsort_t *)b;
8499
8500     comp = cga->nsc - cgb->nsc;
8501     if (comp == 0)
8502     {
8503         comp = cga->ind_gl - cgb->ind_gl;
8504     }
8505
8506     return comp;
8507 }
8508
8509 static void order_int_cg(int n,const gmx_cgsort_t *sort,
8510                          int *a,int *buf)
8511 {
8512     int i;
8513
8514     /* Order the data */
8515     for(i=0; i<n; i++)
8516     {
8517         buf[i] = a[sort[i].ind];
8518     }
8519
8520     /* Copy back to the original array */
8521     for(i=0; i<n; i++)
8522     {
8523         a[i] = buf[i];
8524     }
8525 }
8526
8527 static void order_vec_cg(int n,const gmx_cgsort_t *sort,
8528                          rvec *v,rvec *buf)
8529 {
8530     int i;
8531
8532     /* Order the data */
8533     for(i=0; i<n; i++)
8534     {
8535         copy_rvec(v[sort[i].ind],buf[i]);
8536     }
8537
8538     /* Copy back to the original array */
8539     for(i=0; i<n; i++)
8540     {
8541         copy_rvec(buf[i],v[i]);
8542     }
8543 }
8544
8545 static void order_vec_atom(int ncg,const int *cgindex,const gmx_cgsort_t *sort,
8546                            rvec *v,rvec *buf)
8547 {
8548     int a,atot,cg,cg0,cg1,i;
8549
8550     if (cgindex == NULL)
8551     {
8552         /* Avoid the useless loop of the atoms within a cg */
8553         order_vec_cg(ncg,sort,v,buf);
8554
8555         return;
8556     }
8557
8558     /* Order the data */
8559     a = 0;
8560     for(cg=0; cg<ncg; cg++)
8561     {
8562         cg0 = cgindex[sort[cg].ind];
8563         cg1 = cgindex[sort[cg].ind+1];
8564         for(i=cg0; i<cg1; i++)
8565         {
8566             copy_rvec(v[i],buf[a]);
8567             a++;
8568         }
8569     }
8570     atot = a;
8571
8572     /* Copy back to the original array */
8573     for(a=0; a<atot; a++)
8574     {
8575         copy_rvec(buf[a],v[a]);
8576     }
8577 }
8578
8579 static void ordered_sort(int nsort2,gmx_cgsort_t *sort2,
8580                          int nsort_new,gmx_cgsort_t *sort_new,
8581                          gmx_cgsort_t *sort1)
8582 {
8583     int i1,i2,i_new;
8584
8585     /* The new indices are not very ordered, so we qsort them */
8586     qsort_threadsafe(sort_new,nsort_new,sizeof(sort_new[0]),comp_cgsort);
8587
8588     /* sort2 is already ordered, so now we can merge the two arrays */
8589     i1 = 0;
8590     i2 = 0;
8591     i_new = 0;
8592     while(i2 < nsort2 || i_new < nsort_new)
8593     {
8594         if (i2 == nsort2)
8595         {
8596             sort1[i1++] = sort_new[i_new++];
8597         }
8598         else if (i_new == nsort_new)
8599         {
8600             sort1[i1++] = sort2[i2++];
8601         }
8602         else if (sort2[i2].nsc < sort_new[i_new].nsc ||
8603                  (sort2[i2].nsc == sort_new[i_new].nsc &&
8604                   sort2[i2].ind_gl < sort_new[i_new].ind_gl))
8605         {
8606             sort1[i1++] = sort2[i2++];
8607         }
8608         else
8609         {
8610             sort1[i1++] = sort_new[i_new++];
8611         }
8612     }
8613 }
8614
8615 static int dd_sort_order(gmx_domdec_t *dd,t_forcerec *fr,int ncg_home_old)
8616 {
8617     gmx_domdec_sort_t *sort;
8618     gmx_cgsort_t *cgsort,*sort_i;
8619     int  ncg_new,nsort2,nsort_new,i,*a,moved,*ibuf;
8620     int  sort_last,sort_skip;
8621
8622     sort = dd->comm->sort;
8623
8624     a = fr->ns.grid->cell_index;
8625
8626     moved = NSGRID_SIGNAL_MOVED_FAC*fr->ns.grid->ncells;
8627
8628     if (ncg_home_old >= 0)
8629     {
8630         /* The charge groups that remained in the same ns grid cell
8631          * are completely ordered. So we can sort efficiently by sorting
8632          * the charge groups that did move into the stationary list.
8633          */
8634         ncg_new = 0;
8635         nsort2 = 0;
8636         nsort_new = 0;
8637         for(i=0; i<dd->ncg_home; i++)
8638         {
8639             /* Check if this cg did not move to another node */
8640             if (a[i] < moved)
8641             {
8642                 if (i >= ncg_home_old || a[i] != sort->sort[i].nsc)
8643                 {
8644                     /* This cg is new on this node or moved ns grid cell */
8645                     if (nsort_new >= sort->sort_new_nalloc)
8646                     {
8647                         sort->sort_new_nalloc = over_alloc_dd(nsort_new+1);
8648                         srenew(sort->sort_new,sort->sort_new_nalloc);
8649                     }
8650                     sort_i = &(sort->sort_new[nsort_new++]);
8651                 }
8652                 else
8653                 {
8654                     /* This cg did not move */
8655                     sort_i = &(sort->sort2[nsort2++]);
8656                 }
8657                 /* Sort on the ns grid cell indices
8658                  * and the global topology index.
8659                  * index_gl is irrelevant with cell ns,
8660                  * but we set it here anyhow to avoid a conditional.
8661                  */
8662                 sort_i->nsc    = a[i];
8663                 sort_i->ind_gl = dd->index_gl[i];
8664                 sort_i->ind    = i;
8665                 ncg_new++;
8666             }
8667         }
8668         if (debug)
8669         {
8670             fprintf(debug,"ordered sort cgs: stationary %d moved %d\n",
8671                     nsort2,nsort_new);
8672         }
8673         /* Sort efficiently */
8674         ordered_sort(nsort2,sort->sort2,nsort_new,sort->sort_new,
8675                      sort->sort);
8676     }
8677     else
8678     {
8679         cgsort = sort->sort;
8680         ncg_new = 0;
8681         for(i=0; i<dd->ncg_home; i++)
8682         {
8683             /* Sort on the ns grid cell indices
8684              * and the global topology index
8685              */
8686             cgsort[i].nsc    = a[i];
8687             cgsort[i].ind_gl = dd->index_gl[i];
8688             cgsort[i].ind    = i;
8689             if (cgsort[i].nsc < moved)
8690             {
8691                 ncg_new++;
8692             }
8693         }
8694         if (debug)
8695         {
8696             fprintf(debug,"qsort cgs: %d new home %d\n",dd->ncg_home,ncg_new);
8697         }
8698         /* Determine the order of the charge groups using qsort */
8699         qsort_threadsafe(cgsort,dd->ncg_home,sizeof(cgsort[0]),comp_cgsort);
8700     }
8701
8702     return ncg_new;
8703 }
8704
8705 static int dd_sort_order_nbnxn(gmx_domdec_t *dd,t_forcerec *fr)
8706 {
8707     gmx_cgsort_t *sort;
8708     int  ncg_new,i,*a,na;
8709
8710     sort = dd->comm->sort->sort;
8711
8712     nbnxn_get_atomorder(fr->nbv->nbs,&a,&na);
8713
8714     ncg_new = 0;
8715     for(i=0; i<na; i++)
8716     {
8717         if (a[i] >= 0)
8718         {
8719             sort[ncg_new].ind = a[i];
8720             ncg_new++;
8721         }
8722     }
8723
8724     return ncg_new;
8725 }
8726
8727 static void dd_sort_state(gmx_domdec_t *dd,int ePBC,
8728                           rvec *cgcm,t_forcerec *fr,t_state *state,
8729                           int ncg_home_old)
8730 {
8731     gmx_domdec_sort_t *sort;
8732     gmx_cgsort_t *cgsort,*sort_i;
8733     int  *cgindex;
8734     int  ncg_new,i,*ibuf,cgsize;
8735     rvec *vbuf;
8736
8737     sort = dd->comm->sort;
8738
8739     if (dd->ncg_home > sort->sort_nalloc)
8740     {
8741         sort->sort_nalloc = over_alloc_dd(dd->ncg_home);
8742         srenew(sort->sort,sort->sort_nalloc);
8743         srenew(sort->sort2,sort->sort_nalloc);
8744     }
8745     cgsort = sort->sort;
8746
8747     switch (fr->cutoff_scheme)
8748     {
8749     case ecutsGROUP:
8750         ncg_new = dd_sort_order(dd,fr,ncg_home_old);
8751         break;
8752     case ecutsVERLET:
8753         ncg_new = dd_sort_order_nbnxn(dd,fr);
8754         break;
8755     default:
8756         gmx_incons("unimplemented");
8757         ncg_new = 0;
8758     }
8759
8760     /* We alloc with the old size, since cgindex is still old */
8761     vec_rvec_check_alloc(&dd->comm->vbuf,dd->cgindex[dd->ncg_home]);
8762     vbuf = dd->comm->vbuf.v;
8763
8764     if (dd->comm->bCGs)
8765     {
8766         cgindex = dd->cgindex;
8767     }
8768     else
8769     {
8770         cgindex = NULL;
8771     }
8772
8773     /* Remove the charge groups which are no longer at home here */
8774     dd->ncg_home = ncg_new;
8775     if (debug)
8776     {
8777         fprintf(debug,"Set the new home charge group count to %d\n",
8778                 dd->ncg_home);
8779     }
8780
8781     /* Reorder the state */
8782     for(i=0; i<estNR; i++)
8783     {
8784         if (EST_DISTR(i) && (state->flags & (1<<i)))
8785         {
8786             switch (i)
8787             {
8788             case estX:
8789                 order_vec_atom(dd->ncg_home,cgindex,cgsort,state->x,vbuf);
8790                 break;
8791             case estV:
8792                 order_vec_atom(dd->ncg_home,cgindex,cgsort,state->v,vbuf);
8793                 break;
8794             case estSDX:
8795                 order_vec_atom(dd->ncg_home,cgindex,cgsort,state->sd_X,vbuf);
8796                 break;
8797             case estCGP:
8798                 order_vec_atom(dd->ncg_home,cgindex,cgsort,state->cg_p,vbuf);
8799                 break;
8800             case estLD_RNG:
8801             case estLD_RNGI:
8802             case estDISRE_INITF:
8803             case estDISRE_RM3TAV:
8804             case estORIRE_INITF:
8805             case estORIRE_DTAV:
8806                 /* No ordering required */
8807                 break;
8808             default:
8809                 gmx_incons("Unknown state entry encountered in dd_sort_state");
8810                 break;
8811             }
8812         }
8813     }
8814     if (fr->cutoff_scheme == ecutsGROUP)
8815     {
8816         /* Reorder cgcm */
8817         order_vec_cg(dd->ncg_home,cgsort,cgcm,vbuf);
8818     }
8819
8820     if (dd->ncg_home+1 > sort->ibuf_nalloc)
8821     {
8822         sort->ibuf_nalloc = over_alloc_dd(dd->ncg_home+1);
8823         srenew(sort->ibuf,sort->ibuf_nalloc);
8824     }
8825     ibuf = sort->ibuf;
8826     /* Reorder the global cg index */
8827     order_int_cg(dd->ncg_home,cgsort,dd->index_gl,ibuf);
8828     /* Reorder the cginfo */
8829     order_int_cg(dd->ncg_home,cgsort,fr->cginfo,ibuf);
8830     /* Rebuild the local cg index */
8831     if (dd->comm->bCGs)
8832     {
8833         ibuf[0] = 0;
8834         for(i=0; i<dd->ncg_home; i++)
8835         {
8836             cgsize = dd->cgindex[cgsort[i].ind+1] - dd->cgindex[cgsort[i].ind];
8837             ibuf[i+1] = ibuf[i] + cgsize;
8838         }
8839         for(i=0; i<dd->ncg_home+1; i++)
8840         {
8841             dd->cgindex[i] = ibuf[i];
8842         }
8843     }
8844     else
8845     {
8846         for(i=0; i<dd->ncg_home+1; i++)
8847         {
8848             dd->cgindex[i] = i;
8849         }
8850     }
8851     /* Set the home atom number */
8852     dd->nat_home = dd->cgindex[dd->ncg_home];
8853
8854     if (fr->cutoff_scheme == ecutsVERLET)
8855     {
8856         /* The atoms are now exactly in grid order, update the grid order */
8857         nbnxn_set_atomorder(fr->nbv->nbs);
8858     }
8859     else
8860     {
8861         /* Copy the sorted ns cell indices back to the ns grid struct */
8862         for(i=0; i<dd->ncg_home; i++)
8863         {
8864             fr->ns.grid->cell_index[i] = cgsort[i].nsc;
8865         }
8866         fr->ns.grid->nr = dd->ncg_home;
8867     }
8868 }
8869
8870 static void add_dd_statistics(gmx_domdec_t *dd)
8871 {
8872     gmx_domdec_comm_t *comm;
8873     int ddnat;
8874
8875     comm = dd->comm;
8876
8877     for(ddnat=ddnatZONE; ddnat<ddnatNR; ddnat++)
8878     {
8879         comm->sum_nat[ddnat-ddnatZONE] +=
8880             comm->nat[ddnat] - comm->nat[ddnat-1];
8881     }
8882     comm->ndecomp++;
8883 }
8884
8885 void reset_dd_statistics_counters(gmx_domdec_t *dd)
8886 {
8887     gmx_domdec_comm_t *comm;
8888     int ddnat;
8889
8890     comm = dd->comm;
8891
8892     /* Reset all the statistics and counters for total run counting */
8893     for(ddnat=ddnatZONE; ddnat<ddnatNR; ddnat++)
8894     {
8895         comm->sum_nat[ddnat-ddnatZONE] = 0;
8896     }
8897     comm->ndecomp = 0;
8898     comm->nload = 0;
8899     comm->load_step = 0;
8900     comm->load_sum = 0;
8901     comm->load_max = 0;
8902     clear_ivec(comm->load_lim);
8903     comm->load_mdf = 0;
8904     comm->load_pme = 0;
8905 }
8906
8907 void print_dd_statistics(t_commrec *cr,t_inputrec *ir,FILE *fplog)
8908 {
8909     gmx_domdec_comm_t *comm;
8910     int ddnat;
8911     double av;
8912
8913     comm = cr->dd->comm;
8914
8915     gmx_sumd(ddnatNR-ddnatZONE,comm->sum_nat,cr);
8916
8917     if (fplog == NULL)
8918     {
8919         return;
8920     }
8921
8922     fprintf(fplog,"\n    D O M A I N   D E C O M P O S I T I O N   S T A T I S T I C S\n\n");
8923
8924     for(ddnat=ddnatZONE; ddnat<ddnatNR; ddnat++)
8925     {
8926         av = comm->sum_nat[ddnat-ddnatZONE]/comm->ndecomp;
8927         switch(ddnat)
8928         {
8929         case ddnatZONE:
8930             fprintf(fplog,
8931                     " av. #atoms communicated per step for force:  %d x %.1f\n",
8932                     2,av);
8933             break;
8934         case ddnatVSITE:
8935             if (cr->dd->vsite_comm)
8936             {
8937                 fprintf(fplog,
8938                         " av. #atoms communicated per step for vsites: %d x %.1f\n",
8939                         (EEL_PME(ir->coulombtype) || ir->coulombtype==eelEWALD) ? 3 : 2,
8940                         av);
8941             }
8942             break;
8943         case ddnatCON:
8944             if (cr->dd->constraint_comm)
8945             {
8946                 fprintf(fplog,
8947                         " av. #atoms communicated per step for LINCS:  %d x %.1f\n",
8948                         1 + ir->nLincsIter,av);
8949             }
8950             break;
8951         default:
8952             gmx_incons(" Unknown type for DD statistics");
8953         }
8954     }
8955     fprintf(fplog,"\n");
8956
8957     if (comm->bRecordLoad && EI_DYNAMICS(ir->eI))
8958     {
8959         print_dd_load_av(fplog,cr->dd);
8960     }
8961 }
8962
8963 void dd_partition_system(FILE            *fplog,
8964                          gmx_large_int_t      step,
8965                          t_commrec       *cr,
8966                          gmx_bool            bMasterState,
8967                          int             nstglobalcomm,
8968                          t_state         *state_global,
8969                          gmx_mtop_t      *top_global,
8970                          t_inputrec      *ir,
8971                          t_state         *state_local,
8972                          rvec            **f,
8973                          t_mdatoms       *mdatoms,
8974                          gmx_localtop_t  *top_local,
8975                          t_forcerec      *fr,
8976                          gmx_vsite_t     *vsite,
8977                          gmx_shellfc_t   shellfc,
8978                          gmx_constr_t    constr,
8979                          t_nrnb          *nrnb,
8980                          gmx_wallcycle_t wcycle,
8981                          gmx_bool            bVerbose)
8982 {
8983     gmx_domdec_t *dd;
8984     gmx_domdec_comm_t *comm;
8985     gmx_ddbox_t ddbox={0};
8986     t_block *cgs_gl;
8987     gmx_large_int_t step_pcoupl;
8988     rvec cell_ns_x0,cell_ns_x1;
8989     int  i,j,n,cg0=0,ncg_home_old=-1,ncg_moved,nat_f_novirsum;
8990     gmx_bool bBoxChanged,bNStGlobalComm,bDoDLB,bCheckDLB,bTurnOnDLB,bLogLoad;
8991     gmx_bool bRedist,bSortCG,bResortAll;
8992     ivec ncells_old={0,0,0},ncells_new={0,0,0},np;
8993     real grid_density;
8994     char sbuf[22];
8995
8996     dd = cr->dd;
8997     comm = dd->comm;
8998
8999     bBoxChanged = (bMasterState || DEFORM(*ir));
9000     if (ir->epc != epcNO)
9001     {
9002         /* With nstpcouple > 1 pressure coupling happens.
9003          * one step after calculating the pressure.
9004          * Box scaling happens at the end of the MD step,
9005          * after the DD partitioning.
9006          * We therefore have to do DLB in the first partitioning
9007          * after an MD step where P-coupling occured.
9008          * We need to determine the last step in which p-coupling occurred.
9009          * MRS -- need to validate this for vv?
9010          */
9011         n = ir->nstpcouple;
9012         if (n == 1)
9013         {
9014             step_pcoupl = step - 1;
9015         }
9016         else
9017         {
9018             step_pcoupl = ((step - 1)/n)*n + 1;
9019         }
9020         if (step_pcoupl >= comm->partition_step)
9021         {
9022             bBoxChanged = TRUE;
9023         }
9024     }
9025
9026     bNStGlobalComm = (step % nstglobalcomm == 0);
9027
9028     if (!comm->bDynLoadBal)
9029     {
9030         bDoDLB = FALSE;
9031     }
9032     else
9033     {
9034         /* Should we do dynamic load balacing this step?
9035          * Since it requires (possibly expensive) global communication,
9036          * we might want to do DLB less frequently.
9037          */
9038         if (bBoxChanged || ir->epc != epcNO)
9039         {
9040             bDoDLB = bBoxChanged;
9041         }
9042         else
9043         {
9044             bDoDLB = bNStGlobalComm;
9045         }
9046     }
9047
9048     /* Check if we have recorded loads on the nodes */
9049     if (comm->bRecordLoad && dd_load_count(comm))
9050     {
9051         if (comm->eDLB == edlbAUTO && !comm->bDynLoadBal)
9052         {
9053             /* Check if we should use DLB at the second partitioning
9054              * and every 100 partitionings,
9055              * so the extra communication cost is negligible.
9056              */
9057             n = max(100,nstglobalcomm);
9058             bCheckDLB = (comm->n_load_collect == 0 ||
9059                          comm->n_load_have % n == n-1);
9060         }
9061         else
9062         {
9063             bCheckDLB = FALSE;
9064         }
9065
9066         /* Print load every nstlog, first and last step to the log file */
9067         bLogLoad = ((ir->nstlog > 0 && step % ir->nstlog == 0) ||
9068                     comm->n_load_collect == 0 ||
9069                     (ir->nsteps >= 0 &&
9070                      (step + ir->nstlist > ir->init_step + ir->nsteps)));
9071
9072         /* Avoid extra communication due to verbose screen output
9073          * when nstglobalcomm is set.
9074          */
9075         if (bDoDLB || bLogLoad || bCheckDLB ||
9076             (bVerbose && (ir->nstlist == 0 || nstglobalcomm <= ir->nstlist)))
9077         {
9078             get_load_distribution(dd,wcycle);
9079             if (DDMASTER(dd))
9080             {
9081                 if (bLogLoad)
9082                 {
9083                     dd_print_load(fplog,dd,step-1);
9084                 }
9085                 if (bVerbose)
9086                 {
9087                     dd_print_load_verbose(dd);
9088                 }
9089             }
9090             comm->n_load_collect++;
9091
9092             if (bCheckDLB) {
9093                 /* Since the timings are node dependent, the master decides */
9094                 if (DDMASTER(dd))
9095                 {
9096                     bTurnOnDLB =
9097                         (dd_force_imb_perf_loss(dd) >= DD_PERF_LOSS);
9098                     if (debug)
9099                     {
9100                         fprintf(debug,"step %s, imb loss %f\n",
9101                                 gmx_step_str(step,sbuf),
9102                                 dd_force_imb_perf_loss(dd));
9103                     }
9104                 }
9105                 dd_bcast(dd,sizeof(bTurnOnDLB),&bTurnOnDLB);
9106                 if (bTurnOnDLB)
9107                 {
9108                     turn_on_dlb(fplog,cr,step);
9109                     bDoDLB = TRUE;
9110                 }
9111             }
9112         }
9113         comm->n_load_have++;
9114     }
9115
9116     cgs_gl = &comm->cgs_gl;
9117
9118     bRedist = FALSE;
9119     if (bMasterState)
9120     {
9121         /* Clear the old state */
9122         clear_dd_indices(dd,0,0);
9123
9124         set_ddbox(dd,bMasterState,cr,ir,state_global->box,
9125                   TRUE,cgs_gl,state_global->x,&ddbox);
9126
9127         get_cg_distribution(fplog,step,dd,cgs_gl,
9128                             state_global->box,&ddbox,state_global->x);
9129
9130         dd_distribute_state(dd,cgs_gl,
9131                             state_global,state_local,f);
9132
9133         dd_make_local_cgs(dd,&top_local->cgs);
9134
9135         /* Ensure that we have space for the new distribution */
9136         dd_check_alloc_ncg(fr,state_local,f,dd->ncg_home);
9137
9138         if (fr->cutoff_scheme == ecutsGROUP)
9139         {
9140             calc_cgcm(fplog,0,dd->ncg_home,
9141                       &top_local->cgs,state_local->x,fr->cg_cm);
9142         }
9143
9144         inc_nrnb(nrnb,eNR_CGCM,dd->nat_home);
9145
9146         dd_set_cginfo(dd->index_gl,0,dd->ncg_home,fr,comm->bLocalCG);
9147
9148         cg0 = 0;
9149     }
9150     else if (state_local->ddp_count != dd->ddp_count)
9151     {
9152         if (state_local->ddp_count > dd->ddp_count)
9153         {
9154             gmx_fatal(FARGS,"Internal inconsistency state_local->ddp_count (%d) > dd->ddp_count (%d)",state_local->ddp_count,dd->ddp_count);
9155         }
9156
9157         if (state_local->ddp_count_cg_gl != state_local->ddp_count)
9158         {
9159             gmx_fatal(FARGS,"Internal inconsistency state_local->ddp_count_cg_gl (%d) != state_local->ddp_count (%d)",state_local->ddp_count_cg_gl,state_local->ddp_count);
9160         }
9161
9162         /* Clear the old state */
9163         clear_dd_indices(dd,0,0);
9164
9165         /* Build the new indices */
9166         rebuild_cgindex(dd,cgs_gl->index,state_local);
9167         make_dd_indices(dd,cgs_gl->index,0);
9168
9169         if (fr->cutoff_scheme == ecutsGROUP)
9170         {
9171             /* Redetermine the cg COMs */
9172             calc_cgcm(fplog,0,dd->ncg_home,
9173                       &top_local->cgs,state_local->x,fr->cg_cm);
9174         }
9175
9176         inc_nrnb(nrnb,eNR_CGCM,dd->nat_home);
9177
9178         dd_set_cginfo(dd->index_gl,0,dd->ncg_home,fr,comm->bLocalCG);
9179
9180         set_ddbox(dd,bMasterState,cr,ir,state_local->box,
9181                   TRUE,&top_local->cgs,state_local->x,&ddbox);
9182
9183         bRedist = comm->bDynLoadBal;
9184     }
9185     else
9186     {
9187         /* We have the full state, only redistribute the cgs */
9188
9189         /* Clear the non-home indices */
9190         clear_dd_indices(dd,dd->ncg_home,dd->nat_home);
9191
9192         /* Avoid global communication for dim's without pbc and -gcom */
9193         if (!bNStGlobalComm)
9194         {
9195             copy_rvec(comm->box0    ,ddbox.box0    );
9196             copy_rvec(comm->box_size,ddbox.box_size);
9197         }
9198         set_ddbox(dd,bMasterState,cr,ir,state_local->box,
9199                   bNStGlobalComm,&top_local->cgs,state_local->x,&ddbox);
9200
9201         bBoxChanged = TRUE;
9202         bRedist = TRUE;
9203     }
9204     /* For dim's without pbc and -gcom */
9205     copy_rvec(ddbox.box0    ,comm->box0    );
9206     copy_rvec(ddbox.box_size,comm->box_size);
9207
9208     set_dd_cell_sizes(dd,&ddbox,dynamic_dd_box(&ddbox,ir),bMasterState,bDoDLB,
9209                       step,wcycle);
9210
9211     if (comm->nstDDDumpGrid > 0 && step % comm->nstDDDumpGrid == 0)
9212     {
9213         write_dd_grid_pdb("dd_grid",step,dd,state_local->box,&ddbox);
9214     }
9215
9216     /* Check if we should sort the charge groups */
9217     if (comm->nstSortCG > 0)
9218     {
9219         bSortCG = (bMasterState ||
9220                    (bRedist && (step % comm->nstSortCG == 0)));
9221     }
9222     else
9223     {
9224         bSortCG = FALSE;
9225     }
9226
9227     ncg_home_old = dd->ncg_home;
9228
9229     ncg_moved = 0;
9230     if (bRedist)
9231     {
9232         wallcycle_sub_start(wcycle,ewcsDD_REDIST);
9233
9234         dd_redistribute_cg(fplog,step,dd,ddbox.tric_dir,
9235                            state_local,f,fr,mdatoms,
9236                            !bSortCG,nrnb,&cg0,&ncg_moved);
9237
9238         wallcycle_sub_stop(wcycle,ewcsDD_REDIST);
9239     }
9240
9241     get_nsgrid_boundaries(ddbox.nboundeddim,state_local->box,
9242                           dd,&ddbox,
9243                           &comm->cell_x0,&comm->cell_x1,
9244                           dd->ncg_home,fr->cg_cm,
9245                           cell_ns_x0,cell_ns_x1,&grid_density);
9246
9247     if (bBoxChanged)
9248     {
9249         comm_dd_ns_cell_sizes(dd,&ddbox,cell_ns_x0,cell_ns_x1,step);
9250     }
9251
9252     switch (fr->cutoff_scheme)
9253     {
9254     case ecutsGROUP:
9255         copy_ivec(fr->ns.grid->n,ncells_old);
9256         grid_first(fplog,fr->ns.grid,dd,&ddbox,fr->ePBC,
9257                    state_local->box,cell_ns_x0,cell_ns_x1,
9258                    fr->rlistlong,grid_density);
9259         break;
9260     case ecutsVERLET:
9261         nbnxn_get_ncells(fr->nbv->nbs,&ncells_old[XX],&ncells_old[YY]);
9262         break;
9263     default:
9264         gmx_incons("unimplemented");
9265     }
9266     /* We need to store tric_dir for dd_get_ns_ranges called from ns.c */
9267     copy_ivec(ddbox.tric_dir,comm->tric_dir);
9268
9269     if (bSortCG)
9270     {
9271         wallcycle_sub_start(wcycle,ewcsDD_GRID);
9272
9273         /* Sort the state on charge group position.
9274          * This enables exact restarts from this step.
9275          * It also improves performance by about 15% with larger numbers
9276          * of atoms per node.
9277          */
9278
9279         /* Fill the ns grid with the home cell,
9280          * so we can sort with the indices.
9281          */
9282         set_zones_ncg_home(dd);
9283
9284         switch (fr->cutoff_scheme)
9285         {
9286         case ecutsVERLET:
9287             set_zones_size(dd,state_local->box,&ddbox,0,1);
9288
9289             nbnxn_put_on_grid(fr->nbv->nbs,fr->ePBC,state_local->box,
9290                               0,
9291                               comm->zones.size[0].bb_x0,
9292                               comm->zones.size[0].bb_x1,
9293                               0,dd->ncg_home,
9294                               comm->zones.dens_zone0,
9295                               fr->cginfo,
9296                               state_local->x,
9297                               ncg_moved,comm->moved,
9298                               fr->nbv->grp[eintLocal].kernel_type,
9299                               fr->nbv->grp[eintLocal].nbat);
9300
9301             nbnxn_get_ncells(fr->nbv->nbs,&ncells_new[XX],&ncells_new[YY]);
9302             break;
9303         case ecutsGROUP:
9304             fill_grid(fplog,&comm->zones,fr->ns.grid,dd->ncg_home,
9305                       0,dd->ncg_home,fr->cg_cm);
9306
9307             copy_ivec(fr->ns.grid->n,ncells_new);
9308             break;
9309         default:
9310             gmx_incons("unimplemented");
9311         }
9312
9313         bResortAll = bMasterState;
9314
9315         /* Check if we can user the old order and ns grid cell indices
9316          * of the charge groups to sort the charge groups efficiently.
9317          */
9318         if (ncells_new[XX] != ncells_old[XX] ||
9319             ncells_new[YY] != ncells_old[YY] ||
9320             ncells_new[ZZ] != ncells_old[ZZ])
9321         {
9322             bResortAll = TRUE;
9323         }
9324
9325         if (debug)
9326         {
9327             fprintf(debug,"Step %s, sorting the %d home charge groups\n",
9328                     gmx_step_str(step,sbuf),dd->ncg_home);
9329         }
9330         dd_sort_state(dd,ir->ePBC,fr->cg_cm,fr,state_local,
9331                       bResortAll ? -1 : ncg_home_old);
9332         /* Rebuild all the indices */
9333         cg0 = 0;
9334         ga2la_clear(dd->ga2la);
9335
9336         wallcycle_sub_stop(wcycle,ewcsDD_GRID);
9337     }
9338
9339     wallcycle_sub_start(wcycle,ewcsDD_SETUPCOMM);
9340
9341     /* Setup up the communication and communicate the coordinates */
9342     setup_dd_communication(dd,state_local->box,&ddbox,fr,state_local,f);
9343
9344     /* Set the indices */
9345     make_dd_indices(dd,cgs_gl->index,cg0);
9346
9347     /* Set the charge group boundaries for neighbor searching */
9348     set_cg_boundaries(&comm->zones);
9349
9350     if (fr->cutoff_scheme == ecutsVERLET)
9351     {
9352         set_zones_size(dd,state_local->box,&ddbox,
9353                        bSortCG ? 1 : 0,comm->zones.n);
9354     }
9355
9356     wallcycle_sub_stop(wcycle,ewcsDD_SETUPCOMM);
9357
9358     /*
9359     write_dd_pdb("dd_home",step,"dump",top_global,cr,
9360                  -1,state_local->x,state_local->box);
9361     */
9362
9363     wallcycle_sub_start(wcycle,ewcsDD_MAKETOP);
9364
9365     /* Extract a local topology from the global topology */
9366     for(i=0; i<dd->ndim; i++)
9367     {
9368         np[dd->dim[i]] = comm->cd[i].np;
9369     }
9370     dd_make_local_top(fplog,dd,&comm->zones,dd->npbcdim,state_local->box,
9371                       comm->cellsize_min,np,
9372                       fr,
9373                       fr->cutoff_scheme==ecutsGROUP ? fr->cg_cm : state_local->x,
9374                       vsite,top_global,top_local);
9375
9376     wallcycle_sub_stop(wcycle,ewcsDD_MAKETOP);
9377
9378     wallcycle_sub_start(wcycle,ewcsDD_MAKECONSTR);
9379
9380     /* Set up the special atom communication */
9381     n = comm->nat[ddnatZONE];
9382     for(i=ddnatZONE+1; i<ddnatNR; i++)
9383     {
9384         switch(i)
9385         {
9386         case ddnatVSITE:
9387             if (vsite && vsite->n_intercg_vsite)
9388             {
9389                 n = dd_make_local_vsites(dd,n,top_local->idef.il);
9390             }
9391             break;
9392         case ddnatCON:
9393             if (dd->bInterCGcons || dd->bInterCGsettles)
9394             {
9395                 /* Only for inter-cg constraints we need special code */
9396                 n = dd_make_local_constraints(dd,n,top_global,fr->cginfo,
9397                                               constr,ir->nProjOrder,
9398                                               top_local->idef.il);
9399             }
9400             break;
9401         default:
9402             gmx_incons("Unknown special atom type setup");
9403         }
9404         comm->nat[i] = n;
9405     }
9406
9407     wallcycle_sub_stop(wcycle,ewcsDD_MAKECONSTR);
9408
9409     wallcycle_sub_start(wcycle,ewcsDD_TOPOTHER);
9410
9411     /* Make space for the extra coordinates for virtual site
9412      * or constraint communication.
9413      */
9414     state_local->natoms = comm->nat[ddnatNR-1];
9415     if (state_local->natoms > state_local->nalloc)
9416     {
9417         dd_realloc_state(state_local,f,state_local->natoms);
9418     }
9419
9420     if (fr->bF_NoVirSum)
9421     {
9422         if (vsite && vsite->n_intercg_vsite)
9423         {
9424             nat_f_novirsum = comm->nat[ddnatVSITE];
9425         }
9426         else
9427         {
9428             if (EEL_FULL(ir->coulombtype) && dd->n_intercg_excl > 0)
9429             {
9430                 nat_f_novirsum = dd->nat_tot;
9431             }
9432             else
9433             {
9434                 nat_f_novirsum = dd->nat_home;
9435             }
9436         }
9437     }
9438     else
9439     {
9440         nat_f_novirsum = 0;
9441     }
9442
9443     /* Set the number of atoms required for the force calculation.
9444      * Forces need to be constrained when using a twin-range setup
9445      * or with energy minimization. For simple simulations we could
9446      * avoid some allocation, zeroing and copying, but this is
9447      * probably not worth the complications ande checking.
9448      */
9449     forcerec_set_ranges(fr,dd->ncg_home,dd->ncg_tot,
9450                         dd->nat_tot,comm->nat[ddnatCON],nat_f_novirsum);
9451
9452     /* We make the all mdatoms up to nat_tot_con.
9453      * We could save some work by only setting invmass
9454      * between nat_tot and nat_tot_con.
9455      */
9456     /* This call also sets the new number of home particles to dd->nat_home */
9457     atoms2md(top_global,ir,
9458              comm->nat[ddnatCON],dd->gatindex,0,dd->nat_home,mdatoms);
9459
9460     /* Now we have the charges we can sort the FE interactions */
9461     dd_sort_local_top(dd,mdatoms,top_local);
9462
9463     if (shellfc)
9464     {
9465         /* Make the local shell stuff, currently no communication is done */
9466         make_local_shells(cr,mdatoms,shellfc);
9467     }
9468
9469         if (ir->implicit_solvent)
9470     {
9471         make_local_gb(cr,fr->born,ir->gb_algorithm);
9472     }
9473
9474     init_bonded_thread_force_reduction(fr,&top_local->idef);
9475
9476     if (!(cr->duty & DUTY_PME))
9477     {
9478         /* Send the charges to our PME only node */
9479         gmx_pme_send_q(cr,mdatoms->nChargePerturbed,
9480                        mdatoms->chargeA,mdatoms->chargeB,
9481                        dd_pme_maxshift_x(dd),dd_pme_maxshift_y(dd));
9482     }
9483
9484     if (constr)
9485     {
9486         set_constraints(constr,top_local,ir,mdatoms,cr);
9487     }
9488
9489     if (ir->ePull != epullNO)
9490     {
9491         /* Update the local pull groups */
9492         dd_make_local_pull_groups(dd,ir->pull,mdatoms);
9493     }
9494
9495     if (ir->bRot)
9496     {
9497         /* Update the local rotation groups */
9498         dd_make_local_rotation_groups(dd,ir->rot);
9499     }
9500
9501
9502     add_dd_statistics(dd);
9503
9504     /* Make sure we only count the cycles for this DD partitioning */
9505     clear_dd_cycle_counts(dd);
9506
9507     /* Because the order of the atoms might have changed since
9508      * the last vsite construction, we need to communicate the constructing
9509      * atom coordinates again (for spreading the forces this MD step).
9510      */
9511     dd_move_x_vsites(dd,state_local->box,state_local->x);
9512
9513     wallcycle_sub_stop(wcycle,ewcsDD_TOPOTHER);
9514
9515     if (comm->nstDDDump > 0 && step % comm->nstDDDump == 0)
9516     {
9517         dd_move_x(dd,state_local->box,state_local->x);
9518         write_dd_pdb("dd_dump",step,"dump",top_global,cr,
9519                      -1,state_local->x,state_local->box);
9520     }
9521
9522     /* Store the partitioning step */
9523     comm->partition_step = step;
9524
9525     /* Increase the DD partitioning counter */
9526     dd->ddp_count++;
9527     /* The state currently matches this DD partitioning count, store it */
9528     state_local->ddp_count = dd->ddp_count;
9529     if (bMasterState)
9530     {
9531         /* The DD master node knows the complete cg distribution,
9532          * store the count so we can possibly skip the cg info communication.
9533          */
9534         comm->master_cg_ddp_count = (bSortCG ? 0 : dd->ddp_count);
9535     }
9536
9537     if (comm->DD_debug > 0)
9538     {
9539         /* Set the env var GMX_DD_DEBUG if you suspect corrupted indices */
9540         check_index_consistency(dd,top_global->natoms,ncg_mtop(top_global),
9541                                 "after partitioning");
9542     }
9543 }