src/kernel/repl_ex.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2004, The GROMACS development team,
   6  * check out http://www.gromacs.org for more information.
   7  * Copyright (c) 2012, by the GROMACS development team, led by
   8  * David van der Spoel, Berk Hess, Erik Lindahl, and including many
   9  * others, as listed in the AUTHORS file in the top-level source
  10  * directory and at http://www.gromacs.org.
  11  *
  12  * GROMACS is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU Lesser General Public License
  14  * as published by the Free Software Foundation; either version 2.1
  15  * of the License, or (at your option) any later version.
  16  *
  17  * GROMACS is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20  * Lesser General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public
  23  * License along with GROMACS; if not, see
  24  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  25  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  26  *
  27  * If you want to redistribute modifications to GROMACS, please
  28  * consider that scientific software is very special. Version
  29  * control is crucial - bugs must be traceable. We will be happy to
  30  * consider code for inclusion in the official distribution, but
  31  * derived work must not be called official GROMACS. Details are found
  32  * in the README & COPYING files - if they are missing, get the
  33  * official version at http://www.gromacs.org.
  34  *
  35  * To help us fund GROMACS development, we humbly ask that you cite
  36  * the research papers on the package. Check out http://www.gromacs.org.
  37  */
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42 #include <math.h>
  43 #include "repl_ex.h"
  44 #include "network.h"
  45 #include "random.h"
  46 #include "smalloc.h"
  47 #include "physics.h"
  48 #include "copyrite.h"
  49 #include "macros.h"
  50 #include "vec.h"
  51 #include "names.h"
  52 #include "mvdata.h"
  53 #include "domdec.h"
  54 #include "partdec.h"
  55
  56 #define PROBABILITYCUTOFF 100
  57 /* we don't bother evaluating if events are more rare than exp(-100) = 3.7x10^-44 */
  58
  59 enum { ereTEMP, ereLAMBDA, ereENDSINGLE ,ereTL, ereNR };
  60 const char *erename[ereNR] = { "temperature", "lambda", "end_single_marker", "temperature and lambda"};
  61 /* end_single_marker merely notes the end of single variable replica exchange. All types higher than
  62    it are multiple replica exchange methods */
  63 /* Eventually, should add 'pressure', 'temperature and pressure', 'lambda_and_pressure', 'temperature_lambda_pressure'?;
  64    Let's wait until we feel better about the pressure control methods giving exact ensembles.  Right now, we assume constant pressure  */
  65
  66 typedef struct gmx_repl_ex
  67 {
  68     int  repl;
  69     int  nrepl;
  70     real temp;
  71     int  type;
  72     real **q;
  73     gmx_bool bNPT;
  74     real *pres;
  75     int  *ind;
  76     int *allswaps;
  77     int  nst;
  78     int nex;
  79     int  seed;
  80     int  nattempt[2];
  81     real *prob_sum;
  82     int  **nmoves;
  83     int  *nexchange;
  84
  85     /* these are helper arrays for replica exchange; allocated here so they
  86        don't have to be allocated each time */
  87     int *destinations;
  88     int **cyclic;
  89     int **order;
  90     int *tmpswap;
  91     gmx_bool *incycle;
  92     gmx_bool *bEx;
  93
  94     /* helper arrays to hold the quantities that are exchanged */
  95     real *prob;
  96     real *Epot;
  97     real *beta;
  98     real *Vol;
  99     real **de;
 100
 101 } t_gmx_repl_ex;
 102
 103 static gmx_bool repl_quantity(FILE *fplog,const gmx_multisim_t *ms,
 104                               struct gmx_repl_ex *re,int ere,real q)
 105 {
 106     real *qall;
 107     gmx_bool bDiff;
 108     int  i,s;
 109
 110     snew(qall,ms->nsim);
 111     qall[re->repl] = q;
 112     gmx_sum_sim(ms->nsim,qall,ms);
 113
 114     bDiff = FALSE;
 115     for (s=1; s<ms->nsim; s++)
 116     {
 117         if (qall[s] != qall[0])
 118         {
 119             bDiff = TRUE;
 120         }
 121     }
 122
 123     if (bDiff)
 124     {
 125         /* Set the replica exchange type and quantities */
 126         re->type = ere;
 127
 128         snew(re->q[ere],re->nrepl);
 129         for(s=0; s<ms->nsim; s++)
 130         {
 131             re->q[ere][s] = qall[s];
 132         }
 133     }
 134     sfree(qall);
 135     return bDiff;
 136 }
 137
 138 gmx_repl_ex_t init_replica_exchange(FILE *fplog,
 139                                     const gmx_multisim_t *ms,
 140                                     const t_state *state,
 141                                     const t_inputrec *ir,
 142                                     int nst, int nex, int init_seed)
 143 {
 144     real temp,pres;
 145     int  i,j,k;
 146     struct gmx_repl_ex *re;
 147     gmx_bool bTemp;
 148     gmx_bool bLambda=FALSE;
 149
 150     fprintf(fplog,"\nInitializing Replica Exchange\n");
 151
 152     if (ms == NULL || ms->nsim == 1)
 153     {
 154         gmx_fatal(FARGS,"Nothing to exchange with only one replica, maybe you forgot to set the -multi option of mdrun?");
 155     }
 156
 157     snew(re,1);
 158
 159     re->repl     = ms->sim;
 160     re->nrepl    = ms->nsim;
 161     snew(re->q,ereENDSINGLE);
 162
 163     fprintf(fplog,"Repl  There are %d replicas:\n",re->nrepl);
 164
 165     check_multi_int(fplog,ms,state->natoms,"the number of atoms");
 166     check_multi_int(fplog,ms,ir->eI,"the integrator");
 167     check_multi_large_int(fplog,ms,ir->init_step+ir->nsteps,"init_step+nsteps");
 168     check_multi_large_int(fplog,ms,(ir->init_step+nst-1)/nst,
 169                           "first exchange step: init_step/-replex");
 170     check_multi_int(fplog,ms,ir->etc,"the temperature coupling");
 171     check_multi_int(fplog,ms,ir->opts.ngtc,
 172                     "the number of temperature coupling groups");
 173     check_multi_int(fplog,ms,ir->epc,"the pressure coupling");
 174     check_multi_int(fplog,ms,ir->efep,"free energy");
 175     check_multi_int(fplog,ms,ir->fepvals->n_lambda,"number of lambda states");
 176
 177     re->temp = ir->opts.ref_t[0];
 178     for(i=1; (i<ir->opts.ngtc); i++)
 179     {
 180         if (ir->opts.ref_t[i] != re->temp)
 181         {
 182             fprintf(fplog,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
 183             fprintf(stderr,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
 184         }
 185     }
 186
 187     re->type = -1;
 188     bTemp = repl_quantity(fplog,ms,re,ereTEMP,re->temp);
 189     if (ir->efep != efepNO)
 190     {
 191         bLambda = repl_quantity(fplog,ms,re,ereLAMBDA,(real)ir->fepvals->init_fep_state);
 192     }
 193     if (re->type == -1)  /* nothing was assigned */
 194     {
 195         gmx_fatal(FARGS,"The properties of the %d systems are all the same, there is nothing to exchange",re->nrepl);
 196     }
 197     if (bLambda && bTemp) {
 198         re->type = ereTL;
 199     }
 200
 201     if (bTemp)
 202     {
 203         please_cite(fplog,"Sugita1999a");
 204         if (ir->epc != epcNO)
 205         {
 206             re->bNPT = TRUE;
 207             fprintf(fplog,"Repl  Using Constant Pressure REMD.\n");
 208             please_cite(fplog,"Okabe2001a");
 209         }
 210         if (ir->etc == etcBERENDSEN)
 211         {
 212             gmx_fatal(FARGS,"REMD with the %s thermostat does not produce correct potential energy distributions, consider using the %s thermostat instead",
 213                       ETCOUPLTYPE(ir->etc),ETCOUPLTYPE(etcVRESCALE));
 214         }
 215     }
 216     if (bLambda) {
 217         if (ir->fepvals->delta_lambda != 0)   /* check this? */
 218         {
 219             gmx_fatal(FARGS,"delta_lambda is not zero");
 220         }
 221     }
 222     if (re->bNPT)
 223     {
 224         snew(re->pres,re->nrepl);
 225         if (ir->epct == epctSURFACETENSION)
 226         {
 227             pres = ir->ref_p[ZZ][ZZ];
 228         }
 229         else
 230         {
 231             pres = 0;
 232             j = 0;
 233             for(i=0; i<DIM; i++)
 234             {
 235                 if (ir->compress[i][i] != 0)
 236                 {
 237                     pres += ir->ref_p[i][i];
 238                     j++;
 239                 }
 240             }
 241             pres /= j;
 242         }
 243         re->pres[re->repl] = pres;
 244         gmx_sum_sim(re->nrepl,re->pres,ms);
 245     }
 246
 247     /* Make an index for increasing replica order */
 248     /* only makes sense if one or the other is varying, not both!
 249        if both are varying, we trust the order the person gave. */
 250     snew(re->ind,re->nrepl);
 251     for(i=0; i<re->nrepl; i++)
 252     {
 253         re->ind[i] = i;
 254     }
 255
 256     if (re->type<ereENDSINGLE) {
 257
 258         for(i=0; i<re->nrepl; i++)
 259         {
 260             for(j=i+1; j<re->nrepl; j++)
 261             {
 262                 if (re->q[re->type][re->ind[j]] < re->q[re->type][re->ind[i]])
 263                 {
 264                     k = re->ind[i];
 265                     re->ind[i] = re->ind[j];
 266                     re->ind[j] = k;
 267                 }
 268                 else if (re->q[re->type][re->ind[j]] == re->q[re->type][re->ind[i]])
 269                 {
 270                     gmx_fatal(FARGS,"Two replicas have identical %ss",erename[re->type]);
 271                 }
 272             }
 273         }
 274     }
 275
 276     /* keep track of all the swaps, starting with the initial placement. */
 277     snew(re->allswaps,re->nrepl);
 278     for(i=0; i<re->nrepl; i++)
 279     {
 280         re->allswaps[i] = re->ind[i];
 281     }
 282
 283     switch (re->type)
 284     {
 285     case ereTEMP:
 286         fprintf(fplog,"\nReplica exchange in temperature\n");
 287         for(i=0; i<re->nrepl; i++)
 288         {
 289             fprintf(fplog," %5.1f",re->q[re->type][re->ind[i]]);
 290         }
 291         fprintf(fplog,"\n");
 292         break;
 293     case ereLAMBDA:
 294         fprintf(fplog,"\nReplica exchange in lambda\n");
 295         for(i=0; i<re->nrepl; i++)
 296         {
 297             fprintf(fplog," %3d",(int)re->q[re->type][re->ind[i]]);
 298         }
 299         fprintf(fplog,"\n");
 300         break;
 301     case ereTL:
 302         fprintf(fplog,"\nReplica exchange in temperature and lambda state\n");
 303         for(i=0; i<re->nrepl; i++)
 304         {
 305             fprintf(fplog," %5.1f",re->q[ereTEMP][re->ind[i]]);
 306         }
 307         fprintf(fplog,"\n");
 308         for(i=0; i<re->nrepl; i++)
 309         {
 310             fprintf(fplog," %5d",(int)re->q[ereLAMBDA][re->ind[i]]);
 311         }
 312         fprintf(fplog,"\n");
 313         break;
 314     default:
 315         gmx_incons("Unknown replica exchange quantity");
 316     }
 317     if (re->bNPT)
 318     {
 319         fprintf(fplog,"\nRepl  p");
 320         for(i=0; i<re->nrepl; i++)
 321         {
 322             fprintf(fplog," %5.2f",re->pres[re->ind[i]]);
 323         }
 324
 325         for(i=0; i<re->nrepl; i++)
 326         {
 327             if ((i > 0) && (re->pres[re->ind[i]] < re->pres[re->ind[i-1]]))
 328             {
 329                 fprintf(fplog,"\nWARNING: The reference pressures decrease with increasing temperatures\n\n");
 330                 fprintf(stderr,"\nWARNING: The reference pressures decrease with increasing temperatures\n\n");
 331             }
 332         }
 333     }
 334     re->nst = nst;
 335     if (init_seed == -1)
 336     {
 337         if (MASTERSIM(ms))
 338         {
 339             re->seed = make_seed();
 340         }
 341         else
 342         {
 343             re->seed = 0;
 344         }
 345         gmx_sumi_sim(1,&(re->seed),ms);
 346     }
 347     else
 348     {
 349         re->seed = init_seed;
 350     }
 351     fprintf(fplog,"\nReplica exchange interval: %d\n",re->nst);
 352     fprintf(fplog,"\nReplica random seed: %d\n",re->seed);
 353
 354     re->nattempt[0] = 0;
 355     re->nattempt[1] = 0;
 356
 357     snew(re->prob_sum,re->nrepl);
 358     snew(re->nexchange,re->nrepl);
 359     snew(re->nmoves,re->nrepl);
 360     for (i=0;i<re->nrepl;i++)
 361     {
 362         snew(re->nmoves[i],re->nrepl);
 363     }
 364     fprintf(fplog,"Replica exchange information below: x=exchange, pr=probability\n");
 365
 366     /* generate space for the helper functions so we don't have to snew each time */
 367
 368     snew(re->destinations,re->nrepl);
 369     snew(re->incycle,re->nrepl);
 370     snew(re->tmpswap,re->nrepl);
 371     snew(re->cyclic,re->nrepl);
 372     snew(re->order,re->nrepl);
 373     for (i=0;i<re->nrepl;i++)
 374     {
 375         snew(re->cyclic[i],re->nrepl);
 376         snew(re->order[i],re->nrepl);
 377     }
 378     /* allocate space for the functions storing the data for the replicas */
 379     /* not all of these arrays needed in all cases, but they don't take
 380        up much space, since the max size is nrepl**2 */
 381     snew(re->prob,re->nrepl);
 382     snew(re->bEx,re->nrepl);
 383     snew(re->beta,re->nrepl);
 384     snew(re->Vol,re->nrepl);
 385     snew(re->Epot,re->nrepl);
 386     snew(re->de,re->nrepl);
 387     for (i=0;i<re->nrepl;i++)
 388     {
 389         snew(re->de[i],re->nrepl);
 390     }
 391     re->nex = nex;
 392     return re;
 393 }
 394
 395 static void exchange_reals(const gmx_multisim_t *ms,int b,real *v,int n)
 396 {
 397     real *buf;
 398     int  i;
 399
 400     if (v)
 401     {
 402         snew(buf,n);
 403 #ifdef GMX_MPI
 404         /*
 405           MPI_Sendrecv(v,  n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
 406           buf,n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
 407           ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 408         */
 409         {
 410             MPI_Request mpi_req;
 411
 412             MPI_Isend(v,n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
 413                       ms->mpi_comm_masters,&mpi_req);
 414             MPI_Recv(buf,n*sizeof(real),MPI_BYTE,MSRANK(ms,b),0,
 415                      ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 416             MPI_Wait(&mpi_req,MPI_STATUS_IGNORE);
 417         }
 418 #endif
 419         for(i=0; i<n; i++)
 420         {
 421             v[i] = buf[i];
 422         }
 423         sfree(buf);
 424     }
 425 }
 426
 427
 428 static void exchange_ints(const gmx_multisim_t *ms,int b,int *v,int n)
 429 {
 430   int *buf;
 431   int  i;
 432
 433   if (v) {
 434     snew(buf,n);
 435 #ifdef GMX_MPI
 436     /*
 437     MPI_Sendrecv(v,  n*sizeof(int),MPI_BYTE,MSRANK(ms,b),0,
 438                  buf,n*sizeof(int),MPI_BYTE,MSRANK(ms,b),0,
 439                  ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 440     */
 441     {
 442       MPI_Request mpi_req;
 443
 444       MPI_Isend(v,n*sizeof(int),MPI_BYTE,MSRANK(ms,b),0,
 445                 ms->mpi_comm_masters,&mpi_req);
 446       MPI_Recv(buf,n*sizeof(int),MPI_BYTE,MSRANK(ms,b),0,
 447                ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 448       MPI_Wait(&mpi_req,MPI_STATUS_IGNORE);
 449     }
 450 #endif
 451     for(i=0; i<n; i++)
 452     {
 453         v[i] = buf[i];
 454     }
 455     sfree(buf);
 456   }
 457 }
 458
 459 static void exchange_doubles(const gmx_multisim_t *ms,int b,double *v,int n)
 460 {
 461     double *buf;
 462     int  i;
 463
 464     if (v)
 465     {
 466         snew(buf,n);
 467 #ifdef GMX_MPI
 468         /*
 469           MPI_Sendrecv(v,  n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
 470           buf,n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
 471           ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 472         */
 473         {
 474             MPI_Request mpi_req;
 475
 476             MPI_Isend(v,n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
 477                       ms->mpi_comm_masters,&mpi_req);
 478             MPI_Recv(buf,n*sizeof(double),MPI_BYTE,MSRANK(ms,b),0,
 479                      ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 480             MPI_Wait(&mpi_req,MPI_STATUS_IGNORE);
 481         }
 482 #endif
 483         for(i=0; i<n; i++)
 484         {
 485             v[i] = buf[i];
 486         }
 487         sfree(buf);
 488     }
 489 }
 490
 491 static void exchange_rvecs(const gmx_multisim_t *ms,int b,rvec *v,int n)
 492 {
 493     rvec *buf;
 494     int  i;
 495
 496     if (v)
 497     {
 498         snew(buf,n);
 499 #ifdef GMX_MPI
 500         /*
 501           MPI_Sendrecv(v[0],  n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
 502           buf[0],n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
 503           ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 504         */
 505         {
 506             MPI_Request mpi_req;
 507
 508             MPI_Isend(v[0],n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
 509                       ms->mpi_comm_masters,&mpi_req);
 510             MPI_Recv(buf[0],n*sizeof(rvec),MPI_BYTE,MSRANK(ms,b),0,
 511                      ms->mpi_comm_masters,MPI_STATUS_IGNORE);
 512             MPI_Wait(&mpi_req,MPI_STATUS_IGNORE);
 513         }
 514 #endif
 515         for(i=0; i<n; i++)
 516         {
 517             copy_rvec(buf[i],v[i]);
 518         }
 519         sfree(buf);
 520     }
 521 }
 522
 523 static void exchange_state(const gmx_multisim_t *ms,int b,t_state *state)
 524 {
 525     /* When t_state changes, this code should be updated. */
 526     int ngtc,nnhpres;
 527     ngtc = state->ngtc * state->nhchainlength;
 528     nnhpres = state->nnhpres* state->nhchainlength;
 529     exchange_rvecs(ms,b,state->box,DIM);
 530     exchange_rvecs(ms,b,state->box_rel,DIM);
 531     exchange_rvecs(ms,b,state->boxv,DIM);
 532     exchange_reals(ms,b,&(state->veta),1);
 533     exchange_reals(ms,b,&(state->vol0),1);
 534     exchange_rvecs(ms,b,state->svir_prev,DIM);
 535     exchange_rvecs(ms,b,state->fvir_prev,DIM);
 536     exchange_rvecs(ms,b,state->pres_prev,DIM);
 537     exchange_doubles(ms,b,state->nosehoover_xi,ngtc);
 538     exchange_doubles(ms,b,state->nosehoover_vxi,ngtc);
 539     exchange_doubles(ms,b,state->nhpres_xi,nnhpres);
 540     exchange_doubles(ms,b,state->nhpres_vxi,nnhpres);
 541     exchange_doubles(ms,b,state->therm_integral,state->ngtc);
 542     exchange_rvecs(ms,b,state->x,state->natoms);
 543     exchange_rvecs(ms,b,state->v,state->natoms);
 544     exchange_rvecs(ms,b,state->sd_X,state->natoms);
 545 }
 546
 547 static void copy_rvecs(rvec *s,rvec *d,int n)
 548 {
 549     int i;
 550
 551     if (d != NULL)
 552     {
 553         for(i=0; i<n; i++)
 554         {
 555             copy_rvec(s[i],d[i]);
 556         }
 557     }
 558 }
 559
 560 static void copy_doubles(const double *s,double *d,int n)
 561 {
 562     int i;
 563
 564     if (d != NULL)
 565     {
 566         for(i=0; i<n; i++)
 567         {
 568             d[i] = s[i];
 569         }
 570     }
 571 }
 572
 573 static void copy_reals(const real *s,real *d,int n)
 574 {
 575     int i;
 576
 577     if (d != NULL)
 578     {
 579         for(i=0; i<n; i++)
 580         {
 581             d[i] = s[i];
 582         }
 583     }
 584 }
 585
 586 static void copy_ints(const int *s,int *d,int n)
 587 {
 588     int i;
 589
 590     if (d != NULL)
 591     {
 592         for(i=0; i<n; i++)
 593         {
 594             d[i] = s[i];
 595         }
 596     }
 597 }
 598
 599 #define scopy_rvecs(v,n)   copy_rvecs(state->v,state_local->v,n);
 600 #define scopy_doubles(v,n) copy_doubles(state->v,state_local->v,n);
 601 #define scopy_reals(v,n) copy_reals(state->v,state_local->v,n);
 602 #define scopy_ints(v,n)   copy_ints(state->v,state_local->v,n);
 603
 604 static void copy_state_nonatomdata(t_state *state,t_state *state_local)
 605 {
 606     /* When t_state changes, this code should be updated. */
 607     int ngtc,nnhpres;
 608     ngtc = state->ngtc * state->nhchainlength;
 609     nnhpres = state->nnhpres* state->nhchainlength;
 610     scopy_rvecs(box,DIM);
 611     scopy_rvecs(box_rel,DIM);
 612     scopy_rvecs(boxv,DIM);
 613     state_local->veta = state->veta;
 614     state_local->vol0 = state->vol0;
 615     scopy_rvecs(svir_prev,DIM);
 616     scopy_rvecs(fvir_prev,DIM);
 617     scopy_rvecs(pres_prev,DIM);
 618     scopy_doubles(nosehoover_xi,ngtc);
 619     scopy_doubles(nosehoover_vxi,ngtc);
 620     scopy_doubles(nhpres_xi,nnhpres);
 621     scopy_doubles(nhpres_vxi,nnhpres);
 622     scopy_doubles(therm_integral,state->ngtc);
 623     scopy_rvecs(x,state->natoms);
 624     scopy_rvecs(v,state->natoms);
 625     scopy_rvecs(sd_X,state->natoms);
 626     copy_ints(&(state->fep_state),&(state_local->fep_state),1);
 627     scopy_reals(lambda,efptNR);
 628 }
 629
 630 static void scale_velocities(t_state *state,real fac)
 631 {
 632     int i;
 633
 634     if (state->v)
 635     {
 636         for(i=0; i<state->natoms; i++)
 637         {
 638             svmul(fac,state->v[i],state->v[i]);
 639         }
 640     }
 641 }
 642
 643 static void pd_collect_state(const t_commrec *cr,t_state *state)
 644 {
 645     int shift;
 646
 647     if (debug)
 648     {
 649         fprintf(debug,"Collecting state before exchange\n");
 650     }
 651     shift = cr->nnodes - cr->npmenodes - 1;
 652     move_rvecs(cr,FALSE,FALSE,GMX_LEFT,GMX_RIGHT,state->x,NULL,shift,NULL);
 653     if (state->v)
 654     {
 655         move_rvecs(cr,FALSE,FALSE,GMX_LEFT,GMX_RIGHT,state->v,NULL,shift,NULL);
 656     }
 657     if (state->sd_X)
 658     {
 659         move_rvecs(cr,FALSE,FALSE,GMX_LEFT,GMX_RIGHT,state->sd_X,NULL,shift,NULL);
 660     }
 661 }
 662
 663 static void print_transition_matrix(FILE *fplog,const char *leg,int n,int **nmoves, int *nattempt)
 664 {
 665     int i,j,ntot;
 666     float Tprint;
 667
 668     ntot = nattempt[0] + nattempt[1];
 669     fprintf(fplog,"\n");
 670     fprintf(fplog,"Repl");
 671     for (i=0;i<n;i++)
 672     {
 673         fprintf(fplog,"    ");  /* put the title closer to the center */
 674     }
 675     fprintf(fplog,"Empirical Transition Matrix\n");
 676
 677     fprintf(fplog,"Repl");
 678     for (i=0;i<n;i++)
 679     {
 680         fprintf(fplog,"%8d",(i+1));
 681     }
 682     fprintf(fplog,"\n");
 683
 684     for (i=0;i<n;i++)
 685     {
 686         fprintf(fplog,"Repl");
 687         for (j=0;j<n;j++)
 688         {
 689             Tprint = 0.0;
 690             if (nmoves[i][j] > 0)
 691             {
 692                 Tprint = nmoves[i][j]/(2.0*ntot);
 693             }
 694             fprintf(fplog,"%8.4f",Tprint);
 695         }
 696         fprintf(fplog,"%3d\n",i);
 697     }
 698 }
 699
 700 static void print_ind(FILE *fplog,const char *leg,int n,int *ind,gmx_bool *bEx)
 701 {
 702     int i;
 703
 704     fprintf(fplog,"Repl %2s %2d",leg,ind[0]);
 705     for(i=1; i<n; i++)
 706     {
 707         fprintf(fplog," %c %2d",(bEx!=0 && bEx[i]) ? 'x' : ' ',ind[i]);
 708     }
 709     fprintf(fplog,"\n");
 710 }
 711
 712 static void print_allswitchind(FILE *fplog,int n,int *ind,int *pind, int *allswaps, int *tmpswap)
 713 {
 714     int i;
 715
 716     for (i=0;i<n;i++)
 717     {
 718         tmpswap[i] = allswaps[i];
 719     }
 720     for (i=0;i<n;i++)
 721     {
 722         allswaps[i] = tmpswap[pind[i]];
 723     }
 724
 725     fprintf(fplog,"\nAccepted Exchanges:   ");
 726     for (i=0;i<n;i++)
 727     {
 728         fprintf(fplog,"%d ",pind[i]);
 729     }
 730     fprintf(fplog,"\n");
 731
 732     fprintf(fplog,"Order After Exchange: ");
 733     for (i=0;i<n;i++)
 734     {
 735         fprintf(fplog,"%d ",allswaps[i]);
 736     }
 737     fprintf(fplog,"\n\n");
 738 }
 739
 740 static void print_prob(FILE *fplog,const char *leg,int n,real *prob)
 741 {
 742     int  i;
 743     char buf[8];
 744
 745     fprintf(fplog,"Repl %2s ",leg);
 746     for(i=1; i<n; i++)
 747     {
 748         if (prob[i] >= 0)
 749         {
 750             sprintf(buf,"%4.2f",prob[i]);
 751             fprintf(fplog,"  %3s",buf[0]=='1' ? "1.0" : buf+1);
 752         }
 753         else
 754         {
 755             fprintf(fplog,"     ");
 756         }
 757     }
 758     fprintf(fplog,"\n");
 759 }
 760
 761 static void print_count(FILE *fplog,const char *leg,int n,int *count)
 762 {
 763     int i;
 764
 765     fprintf(fplog,"Repl %2s ",leg);
 766     for(i=1; i<n; i++)
 767     {
 768         fprintf(fplog," %4d",count[i]);
 769     }
 770     fprintf(fplog,"\n");
 771 }
 772
 773 static real calc_delta(FILE *fplog, gmx_bool bPrint, struct gmx_repl_ex *re, int a, int b, int ap, int bp) {
 774
 775     real ediff,dpV,delta=0;
 776     real *Epot = re->Epot;
 777     real *Vol = re->Vol;
 778     real **de = re->de;
 779     real *beta = re->beta;
 780
 781     /* Two cases; we are permuted and not.  In all cases, setting ap = a and bp = b will reduce
 782        to the non permuted case */
 783
 784     switch (re->type)
 785     {
 786     case ereTEMP:
 787         /*
 788          * Okabe et. al. Chem. Phys. Lett. 335 (2001) 435-439
 789          */
 790         ediff = Epot[b] - Epot[a];
 791         delta = -(beta[bp] - beta[ap])*ediff;
 792         break;
 793     case ereLAMBDA:
 794         /* two cases:  when we are permuted, and not.  */
 795         /* non-permuted:
 796            ediff =  E_new - E_old
 797                  =  [H_b(x_a) + H_a(x_b)] - [H_b(x_b) + H_a(x_a)]
 798                  =  [H_b(x_a) - H_a(x_a)] + [H_a(x_b) - H_b(x_b)]
 799                  =  de[b][a] + de[a][b] */
 800         /* permuted:
 801            ediff =  E_new - E_old
 802                  =  [H_bp(x_a) + H_ap(x_b)] - [H_bp(x_b) + H_ap(x_a)]
 803                  =  [H_bp(x_a) - H_ap(x_a)] + [H_ap(x_b) - H_bp(x_b)]
 804                  =  [H_bp(x_a) - H_a(x_a) + H_a(x_a) - H_ap(x_a)] + [H_ap(x_b) - H_b(x_b) + H_b(x_b) - H_bp(x_b)]
 805                  =  [H_bp(x_a) - H_a(x_a)] - [H_ap(x_a) - H_a(x_a)] + [H_ap(x_b) - H_b(x_b)] - H_bp(x_b) - H_b(x_b)]
 806                  =  (de[bp][a] - de[ap][a]) + (de[ap][b] - de[bp][b])    */
 807         ediff = (de[bp][a] - de[ap][a]) + (de[ap][b] - de[bp][b]);
 808         delta = ediff*beta[a]; /* assume all same temperature in this case */
 809         break;
 810     case ereTL:
 811         /* not permuted:  */
 812         /* delta =  reduced E_new - reduced E_old
 813                  =  [beta_b H_b(x_a) + beta_a H_a(x_b)] - [beta_b H_b(x_b) + beta_a H_a(x_a)]
 814                  =  [beta_b H_b(x_a) - beta_a H_a(x_a)] + [beta_a H_a(x_b) - beta_b H_b(x_b)]
 815                  =  [beta_b dH_b(x_a) + beta_b H_a(x_a) - beta_a H_a(x_a)] +
 816                     [beta_a dH_a(x_b) + beta_a H_b(x_b) - beta_b H_b(x_b)]
 817                  =  [beta_b dH_b(x_a) + [beta_a dH_a(x_b) +
 818                     beta_b (H_a(x_a) - H_b(x_b)]) - beta_a (H_a(x_a) - H_b(x_b))
 819                  =  beta_b dH_b(x_a) + beta_a dH_a(x_b) - (beta_b - beta_a)(H_b(x_b) - H_a(x_a) */
 820         /* delta = beta[b]*de[b][a] + beta[a]*de[a][b] - (beta[b] - beta[a])*(Epot[b] - Epot[a]; */
 821         /* permuted (big breath!) */
 822         /*   delta =  reduced E_new - reduced E_old
 823                  =  [beta_bp H_bp(x_a) + beta_ap H_ap(x_b)] - [beta_bp H_bp(x_b) + beta_ap H_ap(x_a)]
 824                  =  [beta_bp H_bp(x_a) - beta_ap H_ap(x_a)] + [beta_ap H_ap(x_b) - beta_bp H_bp(x_b)]
 825                  =  [beta_bp H_bp(x_a) - beta_ap H_ap(x_a)] + [beta_ap H_ap(x_b) - beta_bp H_bp(x_b)]
 826                     - beta_pb H_a(x_a) + beta_ap H_a(x_a) + beta_pb H_a(x_a) - beta_ap H_a(x_a)
 827                     - beta_ap H_b(x_b) + beta_bp H_b(x_b) + beta_ap H_b(x_b) - beta_bp H_b(x_b)
 828                  =  [(beta_bp H_bp(x_a) - beta_bp H_a(x_a)) - (beta_ap H_ap(x_a) - beta_ap H_a(x_a))] +
 829                     [(beta_ap H_ap(x_b)  - beta_ap H_b(x_b)) - (beta_bp H_bp(x_b) - beta_bp H_b(x_b))]
 830                     + beta_pb H_a(x_a) - beta_ap H_a(x_a) + beta_ap H_b(x_b) - beta_bp H_b(x_b)
 831                  =  [beta_bp (H_bp(x_a) - H_a(x_a)) - beta_ap (H_ap(x_a) - H_a(x_a))] +
 832                     [beta_ap (H_ap(x_b) - H_b(x_b)) - beta_bp (H_bp(x_b) - H_b(x_b))]
 833                     + beta_pb (H_a(x_a) - H_b(x_b))  - beta_ap (H_a(x_a) - H_b(x_b))
 834                  =  ([beta_bp de[bp][a] - beta_ap de[ap][a]) + beta_ap de[ap][b]  - beta_bp de[bp][b])
 835                  + (beta_pb-beta_ap)(H_a(x_a) - H_b(x_b))  */
 836         delta = beta[bp]*(de[bp][a] - de[bp][b]) + beta[ap]*(de[ap][b] - de[ap][a]) - (beta[bp]-beta[ap])*(Epot[b]-Epot[a]);
 837         break;
 838     default:
 839         gmx_incons("Unknown replica exchange quantity");
 840     }
 841     if (bPrint)
 842     {
 843         fprintf(fplog,"Repl %d <-> %d  dE_term = %10.3e (kT)\n",a,b,delta);
 844     }
 845     if (re->bNPT)
 846     {
 847         /* revist the calculation for 5.0.  Might be some improvements. */
 848         dpV = (beta[ap]*re->pres[ap]-beta[bp]*re->pres[bp])*(Vol[b]-Vol[a])/PRESFAC;
 849         if (bPrint)
 850         {
 851             fprintf(fplog,"  dpV = %10.3e  d = %10.3e\nb",dpV,delta + dpV);
 852         }
 853         delta += dpV;
 854     }
 855     return delta;
 856 }
 857
 858 static void
 859 test_for_replica_exchange(FILE *fplog,
 860                           const gmx_multisim_t *ms,
 861                           struct gmx_repl_ex *re,
 862                           gmx_enerdata_t *enerd,
 863                           real vol,
 864                           gmx_large_int_t step,
 865                           real time)
 866 {
 867     int  m,i,j,a,b,ap,bp,i0,i1,tmp;
 868     real ediff=0,delta=0,dpV=0;
 869     gmx_bool bPrint,bMultiEx;
 870     gmx_bool *bEx = re->bEx;
 871     real *prob = re->prob;
 872     int *pind = re->destinations;
 873     gmx_bool bEpot=FALSE;
 874     gmx_bool bDLambda=FALSE;
 875     gmx_bool bVol=FALSE;
 876
 877     bMultiEx = (re->nex > 1);  /* multiple exchanges at each state */
 878     fprintf(fplog,"Replica exchange at step " gmx_large_int_pfmt " time %g\n",step,time);
 879
 880     if (re->bNPT)
 881     {
 882         for (i=0;i<re->nrepl;i++)
 883         {
 884             re->Vol[i] = 0;
 885         }
 886         bVol = TRUE;
 887         re->Vol[re->repl]  = vol;
 888     }
 889     if ((re->type == ereTEMP || re->type == ereTL))
 890     {
 891         for (i=0;i<re->nrepl;i++)
 892         {
 893             re->Epot[i] = 0;
 894         }
 895         bEpot = TRUE;
 896         re->Epot[re->repl] = enerd->term[F_EPOT];
 897         /* temperatures of different states*/
 898         for (i=0;i<re->nrepl;i++)
 899         {
 900             re->beta[i] = 1.0/(re->q[ereTEMP][i]*BOLTZ);
 901         }
 902     }
 903     else
 904     {
 905         for (i=0;i<re->nrepl;i++)
 906         {
 907             re->beta[i] = 1.0/(re->temp*BOLTZ);  /* we have a single temperature */
 908         }
 909     }
 910     if (re->type == ereLAMBDA || re->type == ereTL)
 911     {
 912         bDLambda = TRUE;
 913         /* lambda differences. */
 914         /* de[i][j] is the energy of the jth simulation in the ith Hamiltonian
 915            minus the energy of the jth simulation in the jth Hamiltonian */
 916         for (i=0;i<re->nrepl;i++)
 917         {
 918             for (j=0;j<re->nrepl;j++)
 919             {
 920                 re->de[i][j] = 0;
 921             }
 922         }
 923         for (i=0;i<re->nrepl;i++)
 924         {
 925             re->de[i][re->repl] = (enerd->enerpart_lambda[(int)re->q[ereLAMBDA][i]+1]-enerd->enerpart_lambda[0]);
 926         }
 927     }
 928
 929     /* now actually do the communication */
 930     if (bVol)
 931     {
 932         gmx_sum_sim(re->nrepl,re->Vol,ms);
 933     }
 934     if (bEpot)
 935     {
 936         gmx_sum_sim(re->nrepl,re->Epot,ms);
 937     }
 938     if (bDLambda)
 939     {
 940         for (i=0;i<re->nrepl;i++)
 941         {
 942             gmx_sum_sim(re->nrepl,re->de[i],ms);
 943         }
 944     }
 945
 946     /* make a duplicate set of indices for shuffling */
 947     for(i=0;i<re->nrepl;i++)
 948     {
 949         pind[i] = re->ind[i];
 950     }
 951
 952     if (bMultiEx)
 953     {
 954         /* multiple random switch exchange */
 955         for (i=0;i<re->nex;i++)
 956         {
 957             /* randomly select a pair  */
 958             /* find out which state it is from, and what label that state currently has */
 959             i0 = (int)(re->nrepl*rando(&(re->seed)));
 960             i1 = (int)(re->nrepl*rando(&(re->seed)));
 961             if (i0==i1)
 962             {
 963                 i--;
 964                 continue;  /* got the same pair, back up and do it again */
 965             }
 966
 967             a = re->ind[i0];
 968             b = re->ind[i1];
 969             ap = pind[i0];
 970             bp = pind[i1];
 971
 972             bPrint = FALSE; /* too noisy */
 973             delta = calc_delta(fplog,bPrint,re,a,b,ap,bp); /* calculate the energy difference */
 974
 975             /* we actually only use the first space, since there are actually many switches between pairs. */
 976
 977             if (delta <= 0)
 978             {
 979                 /* accepted */
 980                 prob[0] = 1;
 981                 bEx[0] = TRUE;
 982             }
 983             else
 984             {
 985                 if (delta > PROBABILITYCUTOFF)
 986                 {
 987                     prob[0] = 0;
 988                 }
 989                 else
 990                 {
 991                     prob[0] = exp(-delta);
 992                 }
 993                 /* roll a number to determine if accepted */
 994                 bEx[0] = (rando(&(re->seed)) < prob[0]);
 995             }
 996             re->prob_sum[0] += prob[0];
 997
 998             if (bEx[0])
 999             {
1000                 /* swap the states */
1001                 tmp = pind[i0];
1002                 pind[i0] = pind[i1];
1003                 pind[i1] = tmp;
1004             }
1005         }
1006         re->nattempt[0]++;  /* keep track of total permutation trials here */
1007         print_allswitchind(fplog,re->nrepl,re->ind,pind,re->allswaps,re->tmpswap);
1008     }
1009     else
1010     {
1011         /* standard nearest neighbor replica exchange */
1012         m = (step / re->nst) % 2;
1013         for(i=1; i<re->nrepl; i++)
1014         {
1015             a = re->ind[i-1];
1016             b = re->ind[i];
1017
1018             bPrint = (re->repl==a || re->repl==b);
1019             if (i % 2 == m)
1020             {
1021                 delta = calc_delta(fplog,bPrint,re,a,b,a,b);
1022                 if (delta <= 0) {
1023                     /* accepted */
1024                     prob[i] = 1;
1025                     bEx[i] = TRUE;
1026                 }
1027                 else
1028                 {
1029                     if (delta > PROBABILITYCUTOFF)
1030                     {
1031                         prob[i] = 0;
1032                     }
1033                     else
1034                     {
1035                         prob[i] = exp(-delta);
1036                     }
1037                     /* roll a number to determine if accepted */
1038                     bEx[i] = (rando(&(re->seed)) < prob[i]);
1039                 }
1040                 re->prob_sum[i] += prob[i];
1041
1042                 if (bEx[i])
1043                 {
1044                     /* swap these two */
1045                     tmp = pind[i-1];
1046                     pind[i-1] = pind[i];
1047                     pind[i] = tmp;
1048                     re->nexchange[i]++;  /* statistics for back compatibility */
1049                 }
1050             }
1051             else
1052             {
1053                 prob[i] = -1;
1054                 bEx[i] = FALSE;
1055             }
1056         }
1057         /* print some statistics */
1058         print_ind(fplog,"ex",re->nrepl,re->ind,bEx);
1059         print_prob(fplog,"pr",re->nrepl,prob);
1060         fprintf(fplog,"\n");
1061         re->nattempt[m]++;
1062     }
1063
1064     /* record which moves were made and accepted */
1065     for (i=0;i<re->nrepl;i++)
1066     {
1067         re->nmoves[re->ind[i]][pind[i]] +=1;
1068         re->nmoves[pind[i]][re->ind[i]] +=1;
1069     }
1070 }
1071
1072 static void write_debug_x(t_state *state)
1073 {
1074     int i;
1075
1076     if (debug)
1077     {
1078         for(i=0; i<state->natoms; i+=10)
1079         {
1080             fprintf(debug,"dx %5d %10.5f %10.5f %10.5f\n",i,state->x[i][XX],state->x[i][YY],state->x[i][ZZ]);
1081         }
1082     }
1083 }
1084
1085 static void
1086 cyclic_decomposition(FILE *fplog,
1087                      const int *destinations,
1088                      int **cyclic,
1089                      gmx_bool *incycle,
1090                      const int nrepl,
1091                      int *nswap)
1092 {
1093
1094     int i,j,c,p;
1095     int maxlen = 1;
1096     for (i=0;i<nrepl;i++)
1097     {
1098         incycle[i] = FALSE;
1099     }
1100     for (i=0;i<nrepl;i++)  /* one cycle for each replica */
1101     {
1102         if (incycle[i])
1103         {
1104             cyclic[i][0] = -1;
1105             continue;
1106         }
1107         cyclic[i][0] = i;
1108         incycle[i] = TRUE;
1109         c = 1;
1110         p = i;
1111         for (j=0;j<nrepl;j++)  /* potentially all cycles are part, but we will break first */
1112         {
1113             p = destinations[p]; /* start permuting */
1114             if (p==i)
1115             {
1116                 cyclic[i][c] = -1;
1117                 if (c > maxlen)
1118                 {
1119                     maxlen = c;
1120                 }
1121                 break; /* we've reached the original element, the cycle is complete, and we marked the end. */
1122             }
1123             else
1124             {
1125                 cyclic[i][c] = p;  /* each permutation gives a new member of the cycle */
1126                 incycle[p] = TRUE;
1127                 c++;
1128             }
1129         }
1130     }
1131     *nswap = maxlen - 1;
1132
1133     if (debug)
1134     {
1135         for (i=0;i<nrepl;i++)
1136         {
1137             fprintf(debug,"Cycle %d:",i);
1138             for (j=0;j<nrepl;j++)
1139             {
1140                 if (cyclic[i][j] < 0)
1141                 {
1142                     break;
1143                 }
1144                 fprintf(debug,"%2d",cyclic[i][j]);
1145             }
1146             fprintf(debug,"\n");
1147         }
1148         fflush(debug);
1149     }
1150 }
1151
1152 static void
1153 compute_exchange_order(FILE *fplog,
1154                        int **cyclic,
1155                        int **order,
1156                        const int nrepl,
1157                        const int maxswap)
1158 {
1159     int i,j;
1160
1161     for (j=0;j<maxswap;j++)
1162     {
1163         for (i=0;i<nrepl;i++)
1164         {
1165             if (cyclic[i][j+1] >= 0)
1166             {
1167                 order[cyclic[i][j+1]][j] = cyclic[i][j];
1168                 order[cyclic[i][j]][j] = cyclic[i][j+1];
1169             }
1170         }
1171         for (i=0;i<nrepl;i++)
1172         {
1173             if (order[i][j] < 0)
1174             {
1175                 order[i][j] = i; /* if it's not exchanging, it should stay this round*/
1176             }
1177         }
1178     }
1179
1180     if (debug)
1181     {
1182         fprintf(fplog,"Replica Exchange Order\n");
1183         for (i=0;i<nrepl;i++)
1184         {
1185             fprintf(fplog,"Replica %d:",i);
1186             for (j=0;j<maxswap;j++)
1187             {
1188                 if (order[i][j] < 0) break;
1189                 fprintf(debug,"%2d",order[i][j]);
1190             }
1191             fprintf(fplog,"\n");
1192         }
1193         fflush(fplog);
1194     }
1195 }
1196
1197 static void
1198 prepare_to_do_exchange(FILE *fplog,
1199                        const int *destinations,
1200                        const int replica_id,
1201                        const int nrepl,
1202                        int *maxswap,
1203                        int **order,
1204                        int **cyclic,
1205                        int *incycle,
1206                        gmx_bool *bThisReplicaExchanged)
1207 {
1208     int i,j;
1209     /* Hold the cyclic decomposition of the (multiple) replica
1210      * exchange. */
1211     gmx_bool bAnyReplicaExchanged = FALSE;
1212     *bThisReplicaExchanged = FALSE;
1213
1214     for (i = 0; i < nrepl; i++)
1215     {
1216         if (destinations[i] != i) {
1217             /* only mark as exchanged if the index has been shuffled */
1218             bAnyReplicaExchanged = TRUE;
1219             break;
1220         }
1221     }
1222     if (bAnyReplicaExchanged)
1223     {
1224         /* reinitialize the placeholder arrays */
1225         for (i = 0; i < nrepl; i++)
1226         {
1227             for (j = 0; j < nrepl; j++)
1228             {
1229                 cyclic[i][j] = -1;
1230                 order[i][j] = -1;
1231             }
1232         }
1233
1234         /* Identify the cyclic decomposition of the permutation (very
1235          * fast if neighbor replica exchange). */
1236         cyclic_decomposition(fplog,destinations,cyclic,incycle,nrepl,maxswap);
1237
1238         /* Now translate the decomposition into a replica exchange
1239          * order at each step. */
1240         compute_exchange_order(fplog,cyclic,order,nrepl,*maxswap);
1241
1242         /* Did this replica do any exchange at any point? */
1243         for (j = 0; j < *maxswap; j++)
1244         {
1245             if (replica_id != order[replica_id][j])
1246             {
1247                 *bThisReplicaExchanged = TRUE;
1248                 break;
1249             }
1250         }
1251     }
1252 }
1253
1254 gmx_bool replica_exchange(FILE *fplog,const t_commrec *cr,struct gmx_repl_ex *re,
1255                           t_state *state,gmx_enerdata_t *enerd,
1256                           t_state *state_local,gmx_large_int_t step,real time)
1257 {
1258     int i,j;
1259     int replica_id = 0;
1260     int exchange_partner;
1261     int maxswap = 0;
1262     /* Number of rounds of exchanges needed to deal with any multiple
1263      * exchanges. */
1264     /* Where each replica ends up after the exchange attempt(s). */
1265     /* The order in which multiple exchanges will occur. */
1266     gmx_bool bThisReplicaExchanged = FALSE;
1267
1268     if (MASTER(cr))
1269     {
1270         replica_id  = re->repl;
1271         test_for_replica_exchange(fplog,cr->ms,re,enerd,det(state_local->box),step,time);
1272         prepare_to_do_exchange(fplog,re->destinations,replica_id,re->nrepl,&maxswap,
1273                                re->order,re->cyclic,re->incycle,&bThisReplicaExchanged);
1274     }
1275     /* Do intra-simulation broadcast so all processors belonging to
1276      * each simulation know whether they need to participate in
1277      * collecting the state. Otherwise, they might as well get on with
1278      * the next thing to do. */
1279     if (PAR(cr))
1280     {
1281 #ifdef GMX_MPI
1282         MPI_Bcast(&bThisReplicaExchanged,sizeof(gmx_bool),MPI_BYTE,MASTERRANK(cr),
1283                   cr->mpi_comm_mygroup);
1284 #endif
1285     }
1286
1287     if (bThisReplicaExchanged)
1288     {
1289         /* Exchange the states */
1290
1291         if (PAR(cr))
1292         {
1293             /* Collect the global state on the master node */
1294             if (DOMAINDECOMP(cr))
1295             {
1296                 dd_collect_state(cr->dd,state_local,state);
1297             }
1298             else
1299             {
1300                 pd_collect_state(cr,state);
1301             }
1302         }
1303
1304         if (MASTER(cr))
1305         {
1306             /* There will be only one swap cycle with standard replica
1307              * exchange, but there may be multiple swap cycles if we
1308              * allow multiple swaps. */
1309             for (j = 0; j < maxswap; j++)
1310             {
1311                 exchange_partner = re->order[replica_id][j];
1312
1313                 if (exchange_partner != replica_id)
1314                 {
1315                     /* Exchange the global states between the master nodes */
1316                     if (debug)
1317                     {
1318                         fprintf(debug,"Exchanging %d with %d\n",replica_id,exchange_partner);
1319                     }
1320                     exchange_state(cr->ms,exchange_partner,state);
1321                 }
1322             }
1323             /* For temperature-type replica exchange, we need to scale
1324              * the velocities. */
1325             if (re->type == ereTEMP || re->type == ereTL)
1326             {
1327                 scale_velocities(state,sqrt(re->q[ereTEMP][replica_id]/re->q[ereTEMP][re->destinations[replica_id]]));
1328             }
1329
1330         }
1331
1332         /* With domain decomposition the global state is distributed later */
1333         if (!DOMAINDECOMP(cr))
1334         {
1335             /* Copy the global state to the local state data structure */
1336             copy_state_nonatomdata(state,state_local);
1337
1338             if (PAR(cr))
1339             {
1340                 bcast_state(cr,state,FALSE);
1341             }
1342         }
1343     }
1344
1345     return bThisReplicaExchanged;
1346 }
1347
1348 void print_replica_exchange_statistics(FILE *fplog,struct gmx_repl_ex *re)
1349 {
1350     int  i;
1351
1352     fprintf(fplog,"\nReplica exchange statistics\n");
1353
1354     if (re->nex == 0)
1355     {
1356         fprintf(fplog,"Repl  %d attempts, %d odd, %d even\n",
1357                 re->nattempt[0]+re->nattempt[1],re->nattempt[1],re->nattempt[0]);
1358
1359         fprintf(fplog,"Repl  average probabilities:\n");
1360         for(i=1; i<re->nrepl; i++)
1361         {
1362             if (re->nattempt[i%2] == 0)
1363             {
1364                 re->prob[i] = 0;
1365             }
1366             else
1367             {
1368                 re->prob[i] =  re->prob_sum[i]/re->nattempt[i%2];
1369             }
1370         }
1371         print_ind(fplog,"",re->nrepl,re->ind,NULL);
1372         print_prob(fplog,"",re->nrepl,re->prob);
1373
1374         fprintf(fplog,"Repl  number of exchanges:\n");
1375         print_ind(fplog,"",re->nrepl,re->ind,NULL);
1376         print_count(fplog,"",re->nrepl,re->nexchange);
1377
1378         fprintf(fplog,"Repl  average number of exchanges:\n");
1379         for(i=1; i<re->nrepl; i++)
1380         {
1381             if (re->nattempt[i%2] == 0)
1382             {
1383                 re->prob[i] = 0;
1384             }
1385             else
1386             {
1387                 re->prob[i] =  ((real)re->nexchange[i])/re->nattempt[i%2];
1388             }
1389         }
1390         print_ind(fplog,"",re->nrepl,re->ind,NULL);
1391         print_prob(fplog,"",re->nrepl,re->prob);
1392
1393         fprintf(fplog,"\n");
1394     }
1395     /* print the transition matrix */
1396     print_transition_matrix(fplog,"",re->nrepl,re->nmoves,re->nattempt);
1397 }