src/gromacs/mdlib/pme_pp.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2004, The GROMACS development team.
   6  * Copyright (c) 2013,2014, by the GROMACS development team, led by
   7  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   8  * and including many others, as listed in the AUTHORS file in the
   9  * top-level source directory and at http://www.gromacs.org.
  10  *
  11  * GROMACS is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public License
  13  * as published by the Free Software Foundation; either version 2.1
  14  * of the License, or (at your option) any later version.
  15  *
  16  * GROMACS is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with GROMACS; if not, see
  23  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  24  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  *
  26  * If you want to redistribute modifications to GROMACS, please
  27  * consider that scientific software is very special. Version
  28  * control is crucial - bugs must be traceable. We will be happy to
  29  * consider code for inclusion in the official distribution, but
  30  * derived work must not be called official GROMACS. Details are found
  31  * in the README & COPYING files - if they are missing, get the
  32  * official version at http://www.gromacs.org.
  33  *
  34  * To help us fund GROMACS development, we humbly ask that you cite
  35  * the research papers on the package. Check out http://www.gromacs.org.
  36  */
  37
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42
  43 #include <stdio.h>
  44 #include <string.h>
  45 #include <math.h>
  46 #include "typedefs.h"
  47 #include "types/commrec.h"
  48 #include "gromacs/utility/smalloc.h"
  49 #include "gmx_fatal.h"
  50 #include "vec.h"
  51 #include "pme.h"
  52 #include "network.h"
  53 #include "domdec.h"
  54 #include "sighandler.h"
  55
  56 #include "gromacs/utility/gmxmpi.h"
  57
  58 enum {
  59     eCommType_ChargeA, eCommType_ChargeB, eCommType_SQRTC6A, eCommType_SQRTC6B,
  60     eCommType_SigmaA, eCommType_SigmaB, eCommType_NR, eCommType_COORD,
  61     eCommType_CNB
  62 };
  63
  64 /* Some parts of the code(gmx_pme_send_q, gmx_pme_recv_q_x) assume
  65  * that the six first flags are exactly in this order.
  66  * If more PP_PME_...-flags are to be introduced be aware of some of
  67  * the PME-specific flags in pme.h. Currently, they are also passed
  68  * through here.
  69  */
  70
  71 #define PP_PME_CHARGE         (1<<0)
  72 #define PP_PME_CHARGEB        (1<<1)
  73 #define PP_PME_SQRTC6         (1<<2)
  74 #define PP_PME_SQRTC6B        (1<<3)
  75 #define PP_PME_SIGMA          (1<<4)
  76 #define PP_PME_SIGMAB         (1<<5)
  77 #define PP_PME_COORD          (1<<6)
  78 #define PP_PME_FEP_Q          (1<<7)
  79 #define PP_PME_FEP_LJ         (1<<8)
  80 #define PP_PME_ENER_VIR       (1<<9)
  81 #define PP_PME_FINISH         (1<<10)
  82 #define PP_PME_SWITCHGRID     (1<<11)
  83 #define PP_PME_RESETCOUNTERS  (1<<12)
  84
  85 #define PME_PP_SIGSTOP        (1<<0)
  86 #define PME_PP_SIGSTOPNSS     (1<<1)
  87
  88 typedef struct gmx_pme_pp {
  89 #ifdef GMX_MPI
  90     MPI_Comm     mpi_comm_mysim;
  91 #endif
  92     int          nnode;        /* The number of PP node to communicate with  */
  93     int         *node;         /* The PP node ranks                          */
  94     int          node_peer;    /* The peer PP node rank                      */
  95     int         *nat;          /* The number of atom for each PP node        */
  96     int          flags_charge; /* The flags sent along with the last charges */
  97     real        *chargeA;
  98     real        *chargeB;
  99     real        *sqrt_c6A;
 100     real        *sqrt_c6B;
 101     real        *sigmaA;
 102     real        *sigmaB;
 103     rvec        *x;
 104     rvec        *f;
 105     int          nalloc;
 106 #ifdef GMX_MPI
 107     MPI_Request *req;
 108     MPI_Status  *stat;
 109 #endif
 110 } t_gmx_pme_pp;
 111
 112 typedef struct gmx_pme_comm_n_box {
 113     int             natoms;
 114     matrix          box;
 115     int             maxshift_x;
 116     int             maxshift_y;
 117     real            lambda_q;
 118     real            lambda_lj;
 119     int             flags;
 120     gmx_int64_t     step;
 121     ivec            grid_size;    /* For PME grid tuning */
 122     real            ewaldcoeff_q; /* For PME grid tuning */
 123     real            ewaldcoeff_lj;
 124 } gmx_pme_comm_n_box_t;
 125
 126 typedef struct {
 127     matrix          vir_q;
 128     matrix          vir_lj;
 129     real            energy_q;
 130     real            energy_lj;
 131     real            dvdlambda_q;
 132     real            dvdlambda_lj;
 133     float           cycles;
 134     gmx_stop_cond_t stop_cond;
 135 } gmx_pme_comm_vir_ene_t;
 136
 137
 138
 139
 140 gmx_pme_pp_t gmx_pme_pp_init(t_commrec gmx_unused *cr)
 141 {
 142     struct gmx_pme_pp *pme_pp;
 143     int                rank;
 144
 145     snew(pme_pp, 1);
 146
 147 #ifdef GMX_MPI
 148     pme_pp->mpi_comm_mysim = cr->mpi_comm_mysim;
 149     MPI_Comm_rank(cr->mpi_comm_mygroup, &rank);
 150     get_pme_ddnodes(cr, rank, &pme_pp->nnode, &pme_pp->node, &pme_pp->node_peer);
 151     snew(pme_pp->nat, pme_pp->nnode);
 152     snew(pme_pp->req, eCommType_NR*pme_pp->nnode);
 153     snew(pme_pp->stat, eCommType_NR*pme_pp->nnode);
 154     pme_pp->nalloc       = 0;
 155     pme_pp->flags_charge = 0;
 156 #endif
 157
 158     return pme_pp;
 159 }
 160
 161 /* This should be faster with a real non-blocking MPI implementation */
 162 /* #define GMX_PME_DELAYED_WAIT */
 163
 164 static void gmx_pme_send_coeffs_coords_wait(gmx_domdec_t gmx_unused *dd)
 165 {
 166 #ifdef GMX_MPI
 167     if (dd->nreq_pme)
 168     {
 169         MPI_Waitall(dd->nreq_pme, dd->req_pme, MPI_STATUSES_IGNORE);
 170         dd->nreq_pme = 0;
 171     }
 172 #endif
 173 }
 174
 175 static void gmx_pme_send_coeffs_coords(t_commrec *cr, int flags,
 176                                        real gmx_unused *chargeA, real gmx_unused *chargeB,
 177                                        real gmx_unused *c6A, real gmx_unused *c6B,
 178                                        real gmx_unused *sigmaA, real gmx_unused *sigmaB,
 179                                        matrix box, rvec gmx_unused *x,
 180                                        real lambda_q, real lambda_lj,
 181                                        int maxshift_x, int maxshift_y,
 182                                        gmx_int64_t step)
 183 {
 184     gmx_domdec_t         *dd;
 185     gmx_pme_comm_n_box_t *cnb;
 186     int                   n;
 187
 188     dd = cr->dd;
 189     n  = dd->nat_home;
 190
 191     if (debug)
 192     {
 193         fprintf(debug, "PP node %d sending to PME node %d: %d%s%s\n",
 194                 cr->sim_nodeid, dd->pme_nodeid, n,
 195                 flags & PP_PME_CHARGE ? " charges" : "",
 196                 flags & PP_PME_COORD  ? " coordinates" : "");
 197     }
 198
 199 #ifdef GMX_PME_DELAYED_WAIT
 200     /* When can not use cnb until pending communication has finished */
 201     gmx_pme_send_coeffs_coords_wait(dd);
 202 #endif
 203
 204     if (dd->pme_receive_vir_ener)
 205     {
 206         /* Peer PP node: communicate all data */
 207         if (dd->cnb == NULL)
 208         {
 209             snew(dd->cnb, 1);
 210         }
 211         cnb = dd->cnb;
 212
 213         cnb->flags      = flags;
 214         cnb->natoms     = n;
 215         cnb->maxshift_x = maxshift_x;
 216         cnb->maxshift_y = maxshift_y;
 217         cnb->lambda_q   = lambda_q;
 218         cnb->lambda_lj  = lambda_lj;
 219         cnb->step       = step;
 220         if (flags & PP_PME_COORD)
 221         {
 222             copy_mat(box, cnb->box);
 223         }
 224 #ifdef GMX_MPI
 225         MPI_Isend(cnb, sizeof(*cnb), MPI_BYTE,
 226                   dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim,
 227                   &dd->req_pme[dd->nreq_pme++]);
 228 #endif
 229     }
 230     else if (flags & (PP_PME_CHARGE | PP_PME_SQRTC6 | PP_PME_SIGMA))
 231     {
 232 #ifdef GMX_MPI
 233         /* Communicate only the number of atoms */
 234         MPI_Isend(&n, sizeof(n), MPI_BYTE,
 235                   dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim,
 236                   &dd->req_pme[dd->nreq_pme++]);
 237 #endif
 238     }
 239
 240 #ifdef GMX_MPI
 241     if (n > 0)
 242     {
 243         if (flags & PP_PME_CHARGE)
 244         {
 245             MPI_Isend(chargeA, n*sizeof(real), MPI_BYTE,
 246                       dd->pme_nodeid, eCommType_ChargeA, cr->mpi_comm_mysim,
 247                       &dd->req_pme[dd->nreq_pme++]);
 248         }
 249         if (flags & PP_PME_CHARGEB)
 250         {
 251             MPI_Isend(chargeB, n*sizeof(real), MPI_BYTE,
 252                       dd->pme_nodeid, eCommType_ChargeB, cr->mpi_comm_mysim,
 253                       &dd->req_pme[dd->nreq_pme++]);
 254         }
 255         if (flags & PP_PME_SQRTC6)
 256         {
 257             MPI_Isend(c6A, n*sizeof(real), MPI_BYTE,
 258                       dd->pme_nodeid, eCommType_SQRTC6A, cr->mpi_comm_mysim,
 259                       &dd->req_pme[dd->nreq_pme++]);
 260         }
 261         if (flags & PP_PME_SQRTC6B)
 262         {
 263             MPI_Isend(c6B, n*sizeof(real), MPI_BYTE,
 264                       dd->pme_nodeid, eCommType_SQRTC6B, cr->mpi_comm_mysim,
 265                       &dd->req_pme[dd->nreq_pme++]);
 266         }
 267         if (flags & PP_PME_SIGMA)
 268         {
 269             MPI_Isend(sigmaA, n*sizeof(real), MPI_BYTE,
 270                       dd->pme_nodeid, eCommType_SigmaA, cr->mpi_comm_mysim,
 271                       &dd->req_pme[dd->nreq_pme++]);
 272         }
 273         if (flags & PP_PME_SIGMAB)
 274         {
 275             MPI_Isend(sigmaB, n*sizeof(real), MPI_BYTE,
 276                       dd->pme_nodeid, eCommType_SigmaB, cr->mpi_comm_mysim,
 277                       &dd->req_pme[dd->nreq_pme++]);
 278         }
 279         if (flags & PP_PME_COORD)
 280         {
 281             MPI_Isend(x[0], n*sizeof(rvec), MPI_BYTE,
 282                       dd->pme_nodeid, eCommType_COORD, cr->mpi_comm_mysim,
 283                       &dd->req_pme[dd->nreq_pme++]);
 284         }
 285     }
 286
 287 #ifndef GMX_PME_DELAYED_WAIT
 288     /* Wait for the data to arrive */
 289     /* We can skip this wait as we are sure x and q will not be modified
 290      * before the next call to gmx_pme_send_x_q or gmx_pme_receive_f.
 291      */
 292     gmx_pme_send_coeffs_coords_wait(dd);
 293 #endif
 294 #endif
 295 }
 296
 297 void gmx_pme_send_parameters(t_commrec *cr,
 298                              gmx_bool bFreeEnergy_q, gmx_bool bFreeEnergy_lj,
 299                              real *chargeA, real *chargeB,
 300                              real *sqrt_c6A, real *sqrt_c6B,
 301                              real *sigmaA, real *sigmaB,
 302                              int maxshift_x, int maxshift_y)
 303 {
 304     int flags;
 305
 306     /* We always send the charges, even with only LJ- and no Coulomb-PME */
 307     flags = PP_PME_CHARGE;
 308     if (sqrt_c6A != NULL)
 309     {
 310         flags |= PP_PME_SQRTC6;
 311     }
 312     if (sigmaA != NULL)
 313     {
 314         flags |= PP_PME_SIGMA;
 315     }
 316     if (bFreeEnergy_q || bFreeEnergy_lj)
 317     {
 318         /* Assumes that the B state flags are in the bits just above
 319          * the ones for the A state. */
 320         flags |= (flags << 1);
 321     }
 322
 323     gmx_pme_send_coeffs_coords(cr, flags,
 324                                chargeA, chargeB,
 325                                sqrt_c6A, sqrt_c6B, sigmaA, sigmaB,
 326                                NULL, NULL, 0, 0, maxshift_x, maxshift_y, -1);
 327 }
 328
 329 void gmx_pme_send_coordinates(t_commrec *cr, matrix box, rvec *x,
 330                               gmx_bool bFreeEnergy_q, gmx_bool bFreeEnergy_lj,
 331                               real lambda_q, real lambda_lj,
 332                               gmx_bool bEnerVir, int pme_flags,
 333                               gmx_int64_t step)
 334 {
 335     int flags;
 336
 337     flags = pme_flags | PP_PME_COORD;
 338     if (bFreeEnergy_q)
 339     {
 340         flags |= PP_PME_FEP_Q;
 341     }
 342     if (bFreeEnergy_lj)
 343     {
 344         flags |= PP_PME_FEP_LJ;
 345     }
 346     if (bEnerVir)
 347     {
 348         flags |= PP_PME_ENER_VIR;
 349     }
 350     gmx_pme_send_coeffs_coords(cr, flags, NULL, NULL, NULL, NULL, NULL, NULL,
 351                                box, x, lambda_q, lambda_lj, 0, 0, step);
 352 }
 353
 354 void gmx_pme_send_finish(t_commrec *cr)
 355 {
 356     int flags;
 357
 358     flags = PP_PME_FINISH;
 359
 360     gmx_pme_send_coeffs_coords(cr, flags, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0, -1);
 361 }
 362
 363 void gmx_pme_send_switchgrid(t_commrec gmx_unused *cr,
 364                              ivec gmx_unused       grid_size,
 365                              real gmx_unused       ewaldcoeff_q,
 366                              real gmx_unused       ewaldcoeff_lj)
 367 {
 368 #ifdef GMX_MPI
 369     gmx_pme_comm_n_box_t cnb;
 370
 371     /* Only let one PP node signal each PME node */
 372     if (cr->dd->pme_receive_vir_ener)
 373     {
 374         cnb.flags = PP_PME_SWITCHGRID;
 375         copy_ivec(grid_size, cnb.grid_size);
 376         cnb.ewaldcoeff_q  = ewaldcoeff_q;
 377         cnb.ewaldcoeff_lj = ewaldcoeff_lj;
 378
 379         /* We send this, uncommon, message blocking to simplify the code */
 380         MPI_Send(&cnb, sizeof(cnb), MPI_BYTE,
 381                  cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
 382     }
 383 #endif
 384 }
 385
 386 void gmx_pme_send_resetcounters(t_commrec gmx_unused *cr, gmx_int64_t gmx_unused step)
 387 {
 388 #ifdef GMX_MPI
 389     gmx_pme_comm_n_box_t cnb;
 390
 391     /* Only let one PP node signal each PME node */
 392     if (cr->dd->pme_receive_vir_ener)
 393     {
 394         cnb.flags = PP_PME_RESETCOUNTERS;
 395         cnb.step  = step;
 396
 397         /* We send this, uncommon, message blocking to simplify the code */
 398         MPI_Send(&cnb, sizeof(cnb), MPI_BYTE,
 399                  cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
 400     }
 401 #endif
 402 }
 403
 404 int gmx_pme_recv_coeffs_coords(struct gmx_pme_pp          *pme_pp,
 405                                int                        *natoms,
 406                                real                      **chargeA,
 407                                real                      **chargeB,
 408                                real                      **sqrt_c6A,
 409                                real                      **sqrt_c6B,
 410                                real                      **sigmaA,
 411                                real                      **sigmaB,
 412                                matrix gmx_unused           box,
 413                                rvec                      **x,
 414                                rvec                      **f,
 415                                int gmx_unused             *maxshift_x,
 416                                int gmx_unused             *maxshift_y,
 417                                gmx_bool gmx_unused        *bFreeEnergy_q,
 418                                gmx_bool gmx_unused        *bFreeEnergy_lj,
 419                                real gmx_unused            *lambda_q,
 420                                real gmx_unused            *lambda_lj,
 421                                gmx_bool gmx_unused        *bEnerVir,
 422                                int                        *pme_flags,
 423                                gmx_int64_t gmx_unused     *step,
 424                                ivec gmx_unused             grid_size,
 425                                real gmx_unused            *ewaldcoeff_q,
 426                                real gmx_unused            *ewaldcoeff_lj)
 427 {
 428     gmx_pme_comm_n_box_t cnb;
 429     int                  nat = 0, q, messages, sender;
 430     real                *charge_pp;
 431
 432     messages = 0;
 433
 434     /* avoid compiler warning about unused variable without MPI support */
 435     cnb.flags  = 0;
 436     *pme_flags = 0;
 437 #ifdef GMX_MPI
 438     do
 439     {
 440         /* Receive the send count, box and time step from the peer PP node */
 441         MPI_Recv(&cnb, sizeof(cnb), MPI_BYTE,
 442                  pme_pp->node_peer, eCommType_CNB,
 443                  pme_pp->mpi_comm_mysim, MPI_STATUS_IGNORE);
 444
 445         if (debug)
 446         {
 447             fprintf(debug, "PME only node receiving:%s%s%s%s%s\n",
 448                     (cnb.flags & PP_PME_CHARGE)        ? " charges" : "",
 449                     (cnb.flags & PP_PME_COORD )        ? " coordinates" : "",
 450                     (cnb.flags & PP_PME_FINISH)        ? " finish" : "",
 451                     (cnb.flags & PP_PME_SWITCHGRID)    ? " switch grid" : "",
 452                     (cnb.flags & PP_PME_RESETCOUNTERS) ? " reset counters" : "");
 453         }
 454
 455         if (cnb.flags & PP_PME_SWITCHGRID)
 456         {
 457             /* Special case, receive the new parameters and return */
 458             copy_ivec(cnb.grid_size, grid_size);
 459             *ewaldcoeff_q  = cnb.ewaldcoeff_q;
 460             *ewaldcoeff_lj = cnb.ewaldcoeff_lj;
 461             return pmerecvqxSWITCHGRID;
 462         }
 463
 464         if (cnb.flags & PP_PME_RESETCOUNTERS)
 465         {
 466             /* Special case, receive the step and return */
 467             *step = cnb.step;
 468
 469             return pmerecvqxRESETCOUNTERS;
 470         }
 471
 472         if (cnb.flags & (PP_PME_CHARGE | PP_PME_SQRTC6 | PP_PME_SIGMA))
 473         {
 474             /* Receive the send counts from the other PP nodes */
 475             for (sender = 0; sender < pme_pp->nnode; sender++)
 476             {
 477                 if (pme_pp->node[sender] == pme_pp->node_peer)
 478                 {
 479                     pme_pp->nat[sender] = cnb.natoms;
 480                 }
 481                 else
 482                 {
 483                     MPI_Irecv(&(pme_pp->nat[sender]), sizeof(pme_pp->nat[0]),
 484                               MPI_BYTE,
 485                               pme_pp->node[sender], eCommType_CNB,
 486                               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 487                 }
 488             }
 489             MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 490             messages = 0;
 491
 492             nat = 0;
 493             for (sender = 0; sender < pme_pp->nnode; sender++)
 494             {
 495                 nat += pme_pp->nat[sender];
 496             }
 497
 498             if (nat > pme_pp->nalloc)
 499             {
 500                 pme_pp->nalloc = over_alloc_dd(nat);
 501                 if (cnb.flags & PP_PME_CHARGE)
 502                 {
 503                     srenew(pme_pp->chargeA, pme_pp->nalloc);
 504                 }
 505                 if (cnb.flags & PP_PME_CHARGEB)
 506                 {
 507                     srenew(pme_pp->chargeB, pme_pp->nalloc);
 508                 }
 509                 if (cnb.flags & PP_PME_SQRTC6)
 510                 {
 511                     srenew(pme_pp->sqrt_c6A, pme_pp->nalloc);
 512                 }
 513                 if (cnb.flags & PP_PME_SQRTC6B)
 514                 {
 515                     srenew(pme_pp->sqrt_c6B, pme_pp->nalloc);
 516                 }
 517                 if (cnb.flags & PP_PME_SIGMA)
 518                 {
 519                     srenew(pme_pp->sigmaA, pme_pp->nalloc);
 520                 }
 521                 if (cnb.flags & PP_PME_SIGMAB)
 522                 {
 523                     srenew(pme_pp->sigmaB, pme_pp->nalloc);
 524                 }
 525                 srenew(pme_pp->x, pme_pp->nalloc);
 526                 srenew(pme_pp->f, pme_pp->nalloc);
 527             }
 528
 529             /* maxshift is sent when the charges are sent */
 530             *maxshift_x = cnb.maxshift_x;
 531             *maxshift_y = cnb.maxshift_y;
 532
 533             /* Receive the charges in place */
 534             for (q = 0; q < eCommType_NR; q++)
 535             {
 536                 if (!(cnb.flags & (PP_PME_CHARGE<<q)))
 537                 {
 538                     continue;
 539                 }
 540                 switch (q)
 541                 {
 542                     case eCommType_ChargeA: charge_pp = pme_pp->chargeA;  break;
 543                     case eCommType_ChargeB: charge_pp = pme_pp->chargeB;  break;
 544                     case eCommType_SQRTC6A: charge_pp = pme_pp->sqrt_c6A; break;
 545                     case eCommType_SQRTC6B: charge_pp = pme_pp->sqrt_c6B; break;
 546                     case eCommType_SigmaA:  charge_pp = pme_pp->sigmaA;   break;
 547                     case eCommType_SigmaB:  charge_pp = pme_pp->sigmaB;   break;
 548                     default: gmx_incons("Wrong eCommType");
 549                 }
 550                 nat = 0;
 551                 for (sender = 0; sender < pme_pp->nnode; sender++)
 552                 {
 553                     if (pme_pp->nat[sender] > 0)
 554                     {
 555                         MPI_Irecv(charge_pp+nat,
 556                                   pme_pp->nat[sender]*sizeof(real),
 557                                   MPI_BYTE,
 558                                   pme_pp->node[sender], q,
 559                                   pme_pp->mpi_comm_mysim,
 560                                   &pme_pp->req[messages++]);
 561                         nat += pme_pp->nat[sender];
 562                         if (debug)
 563                         {
 564                             fprintf(debug, "Received from PP node %d: %d "
 565                                     "charges\n",
 566                                     pme_pp->node[sender], pme_pp->nat[sender]);
 567                         }
 568                     }
 569                 }
 570             }
 571
 572             pme_pp->flags_charge = cnb.flags;
 573         }
 574
 575         if (cnb.flags & PP_PME_COORD)
 576         {
 577             if (!(pme_pp->flags_charge & (PP_PME_CHARGE | PP_PME_SQRTC6)))
 578             {
 579                 gmx_incons("PME-only node received coordinates before charges and/or C6-values"
 580                            );
 581             }
 582
 583             /* The box, FE flag and lambda are sent along with the coordinates
 584              *  */
 585             copy_mat(cnb.box, box);
 586             *bFreeEnergy_q  = ((cnb.flags & GMX_PME_DO_COULOMB) &&
 587                                (cnb.flags & PP_PME_FEP_Q));
 588             *bFreeEnergy_lj = ((cnb.flags & GMX_PME_DO_LJ) &&
 589                                (cnb.flags & PP_PME_FEP_LJ));
 590             *lambda_q       = cnb.lambda_q;
 591             *lambda_lj      = cnb.lambda_lj;
 592             *bEnerVir       = (cnb.flags & PP_PME_ENER_VIR);
 593             *pme_flags      = cnb.flags;
 594
 595             if (*bFreeEnergy_q && !(pme_pp->flags_charge & PP_PME_CHARGEB))
 596             {
 597                 gmx_incons("PME-only node received free energy request, but "
 598                            "did not receive B-state charges");
 599             }
 600
 601             if (*bFreeEnergy_lj && !(pme_pp->flags_charge & PP_PME_SQRTC6B))
 602             {
 603                 gmx_incons("PME-only node received free energy request, but "
 604                            "did not receive B-state C6-values");
 605             }
 606
 607             /* Receive the coordinates in place */
 608             nat = 0;
 609             for (sender = 0; sender < pme_pp->nnode; sender++)
 610             {
 611                 if (pme_pp->nat[sender] > 0)
 612                 {
 613                     MPI_Irecv(pme_pp->x[nat], pme_pp->nat[sender]*sizeof(rvec),
 614                               MPI_BYTE,
 615                               pme_pp->node[sender], eCommType_COORD,
 616                               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 617                     nat += pme_pp->nat[sender];
 618                     if (debug)
 619                     {
 620                         fprintf(debug, "Received from PP node %d: %d "
 621                                 "coordinates\n",
 622                                 pme_pp->node[sender], pme_pp->nat[sender]);
 623                     }
 624                 }
 625             }
 626         }
 627
 628         /* Wait for the coordinates and/or charges to arrive */
 629         MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 630         messages = 0;
 631     }
 632     while (!(cnb.flags & (PP_PME_COORD | PP_PME_FINISH)));
 633
 634     *step = cnb.step;
 635 #endif
 636
 637     *natoms   = nat;
 638     *chargeA  = pme_pp->chargeA;
 639     *chargeB  = pme_pp->chargeB;
 640     *sqrt_c6A = pme_pp->sqrt_c6A;
 641     *sqrt_c6B = pme_pp->sqrt_c6B;
 642     *sigmaA   = pme_pp->sigmaA;
 643     *sigmaB   = pme_pp->sigmaB;
 644     *x        = pme_pp->x;
 645     *f        = pme_pp->f;
 646
 647     return ((cnb.flags & PP_PME_FINISH) ? pmerecvqxFINISH : pmerecvqxX);
 648 }
 649
 650 static void receive_virial_energy(t_commrec *cr,
 651                                   matrix vir_q, real *energy_q,
 652                                   matrix vir_lj, real *energy_lj,
 653                                   real *dvdlambda_q, real *dvdlambda_lj,
 654                                   float *pme_cycles)
 655 {
 656     gmx_pme_comm_vir_ene_t cve;
 657
 658     if (cr->dd->pme_receive_vir_ener)
 659     {
 660         if (debug)
 661         {
 662             fprintf(debug,
 663                     "PP node %d receiving from PME node %d: virial and energy\n",
 664                     cr->sim_nodeid, cr->dd->pme_nodeid);
 665         }
 666 #ifdef GMX_MPI
 667         MPI_Recv(&cve, sizeof(cve), MPI_BYTE, cr->dd->pme_nodeid, 1, cr->mpi_comm_mysim,
 668                  MPI_STATUS_IGNORE);
 669 #else
 670         memset(&cve, 0, sizeof(cve));
 671 #endif
 672
 673         m_add(vir_q, cve.vir_q, vir_q);
 674         m_add(vir_lj, cve.vir_lj, vir_lj);
 675         *energy_q      = cve.energy_q;
 676         *energy_lj     = cve.energy_lj;
 677         *dvdlambda_q  += cve.dvdlambda_q;
 678         *dvdlambda_lj += cve.dvdlambda_lj;
 679         *pme_cycles    = cve.cycles;
 680
 681         if (cve.stop_cond != gmx_stop_cond_none)
 682         {
 683             gmx_set_stop_condition(cve.stop_cond);
 684         }
 685     }
 686     else
 687     {
 688         *energy_q   = 0;
 689         *energy_lj  = 0;
 690         *pme_cycles = 0;
 691     }
 692 }
 693
 694 void gmx_pme_receive_f(t_commrec *cr,
 695                        rvec f[], matrix vir_q, real *energy_q,
 696                        matrix vir_lj, real *energy_lj,
 697                        real *dvdlambda_q, real *dvdlambda_lj,
 698                        float *pme_cycles)
 699 {
 700     int natoms, i;
 701
 702 #ifdef GMX_PME_DELAYED_WAIT
 703     /* Wait for the x request to finish */
 704     gmx_pme_send_coeffs_coords_wait(cr->dd);
 705 #endif
 706
 707     natoms = cr->dd->nat_home;
 708
 709     if (natoms > cr->dd->pme_recv_f_alloc)
 710     {
 711         cr->dd->pme_recv_f_alloc = over_alloc_dd(natoms);
 712         srenew(cr->dd->pme_recv_f_buf, cr->dd->pme_recv_f_alloc);
 713     }
 714
 715 #ifdef GMX_MPI
 716     MPI_Recv(cr->dd->pme_recv_f_buf[0],
 717              natoms*sizeof(rvec), MPI_BYTE,
 718              cr->dd->pme_nodeid, 0, cr->mpi_comm_mysim,
 719              MPI_STATUS_IGNORE);
 720 #endif
 721
 722     for (i = 0; i < natoms; i++)
 723     {
 724         rvec_inc(f[i], cr->dd->pme_recv_f_buf[i]);
 725     }
 726
 727
 728     receive_virial_energy(cr, vir_q, energy_q, vir_lj, energy_lj, dvdlambda_q, dvdlambda_lj, pme_cycles);
 729 }
 730
 731 void gmx_pme_send_force_vir_ener(struct gmx_pme_pp *pme_pp,
 732                                  rvec gmx_unused *f,
 733                                  matrix vir_q, real energy_q,
 734                                  matrix vir_lj, real energy_lj,
 735                                  real dvdlambda_q, real dvdlambda_lj,
 736                                  float cycles)
 737 {
 738     gmx_pme_comm_vir_ene_t cve;
 739     int                    messages, ind_start, ind_end, receiver;
 740
 741     cve.cycles = cycles;
 742
 743     /* Now the evaluated forces have to be transferred to the PP nodes */
 744     messages = 0;
 745     ind_end  = 0;
 746     for (receiver = 0; receiver < pme_pp->nnode; receiver++)
 747     {
 748         ind_start = ind_end;
 749         ind_end   = ind_start + pme_pp->nat[receiver];
 750 #ifdef GMX_MPI
 751         if (MPI_Isend(f[ind_start], (ind_end-ind_start)*sizeof(rvec), MPI_BYTE,
 752                       pme_pp->node[receiver], 0,
 753                       pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]) != 0)
 754         {
 755             gmx_comm("MPI_Isend failed in do_pmeonly");
 756         }
 757 #endif
 758     }
 759
 760     /* send virial and energy to our last PP node */
 761     copy_mat(vir_q, cve.vir_q);
 762     copy_mat(vir_lj, cve.vir_lj);
 763     cve.energy_q     = energy_q;
 764     cve.energy_lj    = energy_lj;
 765     cve.dvdlambda_q  = dvdlambda_q;
 766     cve.dvdlambda_lj = dvdlambda_lj;
 767     /* check for the signals to send back to a PP node */
 768     cve.stop_cond = gmx_get_stop_condition();
 769
 770     cve.cycles = cycles;
 771
 772     if (debug)
 773     {
 774         fprintf(debug, "PME node sending to PP node %d: virial and energy\n",
 775                 pme_pp->node_peer);
 776     }
 777 #ifdef GMX_MPI
 778     MPI_Isend(&cve, sizeof(cve), MPI_BYTE,
 779               pme_pp->node_peer, 1,
 780               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 781
 782     /* Wait for the forces to arrive */
 783     MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 784 #endif
 785 }