src/gromacs/mdlib/pme_pp.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2004, The GROMACS development team.
   6  * Copyright (c) 2013, by the GROMACS development team, led by
   7  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   8  * and including many others, as listed in the AUTHORS file in the
   9  * top-level source directory and at http://www.gromacs.org.
  10  *
  11  * GROMACS is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public License
  13  * as published by the Free Software Foundation; either version 2.1
  14  * of the License, or (at your option) any later version.
  15  *
  16  * GROMACS is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with GROMACS; if not, see
  23  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  24  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  *
  26  * If you want to redistribute modifications to GROMACS, please
  27  * consider that scientific software is very special. Version
  28  * control is crucial - bugs must be traceable. We will be happy to
  29  * consider code for inclusion in the official distribution, but
  30  * derived work must not be called official GROMACS. Details are found
  31  * in the README & COPYING files - if they are missing, get the
  32  * official version at http://www.gromacs.org.
  33  *
  34  * To help us fund GROMACS development, we humbly ask that you cite
  35  * the research papers on the package. Check out http://www.gromacs.org.
  36  */
  37
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42
  43 #include <stdio.h>
  44 #include <string.h>
  45 #include <math.h>
  46 #include "typedefs.h"
  47 #include "smalloc.h"
  48 #include "gmx_fatal.h"
  49 #include "vec.h"
  50 #include "pme.h"
  51 #include "network.h"
  52 #include "domdec.h"
  53 #include "sighandler.h"
  54
  55 #include "gromacs/utility/gmxmpi.h"
  56
  57 enum {
  58     eCommType_ChargeA, eCommType_ChargeB, eCommType_C6A, eCommType_C6B,
  59     eCommType_SigmaA, eCommType_SigmaB, eCommType_NR, eCommType_COORD,
  60     eCommType_CNB
  61 };
  62
  63 /* Some parts of the code(gmx_pme_send_q, gmx_pme_recv_q_x) assume
  64  * that the six first flags are exactly in this order.
  65  * If more PP_PME_...-flags are to be introduced be aware of some of
  66  * the PME-specific flags in pme.h. Currently, they are also passed
  67  * through here.
  68  */
  69
  70 #define PP_PME_CHARGE         (1<<0)
  71 #define PP_PME_CHARGEB        (1<<1)
  72 #define PP_PME_C6             (1<<2)
  73 #define PP_PME_C6B            (1<<3)
  74 #define PP_PME_SIGMA          (1<<4)
  75 #define PP_PME_SIGMAB         (1<<5)
  76 #define PP_PME_COORD          (1<<6)
  77 #define PP_PME_FEP_Q          (1<<7)
  78 #define PP_PME_FEP_LJ         (1<<8)
  79 #define PP_PME_ENER_VIR       (1<<9)
  80 #define PP_PME_FINISH         (1<<10)
  81 #define PP_PME_SWITCHGRID     (1<<11)
  82 #define PP_PME_RESETCOUNTERS  (1<<12)
  83
  84 #define PME_PP_SIGSTOP        (1<<0)
  85 #define PME_PP_SIGSTOPNSS     (1<<1)
  86
  87 typedef struct gmx_pme_pp {
  88 #ifdef GMX_MPI
  89     MPI_Comm     mpi_comm_mysim;
  90 #endif
  91     int          nnode;        /* The number of PP node to communicate with  */
  92     int         *node;         /* The PP node ranks                          */
  93     int          node_peer;    /* The peer PP node rank                      */
  94     int         *nat;          /* The number of atom for each PP node        */
  95     int          flags_charge; /* The flags sent along with the last charges */
  96     real        *chargeA;
  97     real        *chargeB;
  98     real        *c6A;
  99     real        *c6B;
 100     real        *sigmaA;
 101     real        *sigmaB;
 102     rvec        *x;
 103     rvec        *f;
 104     int          nalloc;
 105 #ifdef GMX_MPI
 106     MPI_Request *req;
 107     MPI_Status  *stat;
 108 #endif
 109 } t_gmx_pme_pp;
 110
 111 typedef struct gmx_pme_comm_n_box {
 112     int             natoms;
 113     matrix          box;
 114     int             maxshift_x;
 115     int             maxshift_y;
 116     real            lambda_q;
 117     real            lambda_lj;
 118     int             flags;
 119     gmx_large_int_t step;
 120     ivec            grid_size;    /* For PME grid tuning */
 121     real            ewaldcoeff_q; /* For PME grid tuning */
 122     real            ewaldcoeff_lj;
 123 } gmx_pme_comm_n_box_t;
 124
 125 typedef struct {
 126     matrix          vir_q;
 127     matrix          vir_lj;
 128     real            energy_q;
 129     real            energy_lj;
 130     real            dvdlambda_q;
 131     real            dvdlambda_lj;
 132     float           cycles;
 133     gmx_stop_cond_t stop_cond;
 134 } gmx_pme_comm_vir_ene_t;
 135
 136
 137
 138
 139 gmx_pme_pp_t gmx_pme_pp_init(t_commrec gmx_unused *cr)
 140 {
 141     struct gmx_pme_pp *pme_pp;
 142     int                rank;
 143
 144     snew(pme_pp, 1);
 145
 146 #ifdef GMX_MPI
 147     pme_pp->mpi_comm_mysim = cr->mpi_comm_mysim;
 148     MPI_Comm_rank(cr->mpi_comm_mygroup, &rank);
 149     get_pme_ddnodes(cr, rank, &pme_pp->nnode, &pme_pp->node, &pme_pp->node_peer);
 150     snew(pme_pp->nat, pme_pp->nnode);
 151     snew(pme_pp->req, eCommType_NR*pme_pp->nnode);
 152     snew(pme_pp->stat, eCommType_NR*pme_pp->nnode);
 153     pme_pp->nalloc       = 0;
 154     pme_pp->flags_charge = 0;
 155 #endif
 156
 157     return pme_pp;
 158 }
 159
 160 /* This should be faster with a real non-blocking MPI implementation */
 161 /* #define GMX_PME_DELAYED_WAIT */
 162
 163 static void gmx_pme_send_params_coords_wait(gmx_domdec_t gmx_unused *dd)
 164 {
 165 #ifdef GMX_MPI
 166     if (dd->nreq_pme)
 167     {
 168         MPI_Waitall(dd->nreq_pme, dd->req_pme, MPI_STATUSES_IGNORE);
 169         dd->nreq_pme = 0;
 170     }
 171 #endif
 172 }
 173
 174 static void gmx_pme_send_params_coords(t_commrec *cr, int flags,
 175                                        real gmx_unused *chargeA, real gmx_unused *chargeB,
 176                                        real gmx_unused *c6A, real gmx_unused *c6B,
 177                                        real gmx_unused *sigmaA, real gmx_unused *sigmaB,
 178                                        matrix box, rvec gmx_unused *x,
 179                                        real lambda_q, real lambda_lj,
 180                                        int maxshift_x, int maxshift_y,
 181                                        gmx_large_int_t step)
 182 {
 183     gmx_domdec_t         *dd;
 184     gmx_pme_comm_n_box_t *cnb;
 185     int                   n;
 186
 187     dd = cr->dd;
 188     n  = dd->nat_home;
 189
 190     if (debug)
 191     {
 192         fprintf(debug, "PP node %d sending to PME node %d: %d%s%s\n",
 193                 cr->sim_nodeid, dd->pme_nodeid, n,
 194                 flags & PP_PME_CHARGE ? " charges" : "",
 195                 flags & PP_PME_COORD  ? " coordinates" : "");
 196     }
 197
 198 #ifdef GMX_PME_DELAYED_WAIT
 199     /* When can not use cnb until pending communication has finished */
 200     gmx_pme_send_params_coords_wait(dd);
 201 #endif
 202
 203     if (dd->pme_receive_vir_ener)
 204     {
 205         /* Peer PP node: communicate all data */
 206         if (dd->cnb == NULL)
 207         {
 208             snew(dd->cnb, 1);
 209         }
 210         cnb = dd->cnb;
 211
 212         cnb->flags      = flags;
 213         cnb->natoms     = n;
 214         cnb->maxshift_x = maxshift_x;
 215         cnb->maxshift_y = maxshift_y;
 216         cnb->lambda_q   = lambda_q;
 217         cnb->lambda_lj  = lambda_lj;
 218         cnb->step       = step;
 219         if (flags & PP_PME_COORD)
 220         {
 221             copy_mat(box, cnb->box);
 222         }
 223 #ifdef GMX_MPI
 224         MPI_Isend(cnb, sizeof(*cnb), MPI_BYTE,
 225                   dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim,
 226                   &dd->req_pme[dd->nreq_pme++]);
 227 #endif
 228     }
 229     else if (flags & (PP_PME_CHARGE | PP_PME_C6 | PP_PME_SIGMA))
 230     {
 231 #ifdef GMX_MPI
 232         /* Communicate only the number of atoms */
 233         MPI_Isend(&n, sizeof(n), MPI_BYTE,
 234                   dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim,
 235                   &dd->req_pme[dd->nreq_pme++]);
 236 #endif
 237     }
 238
 239 #ifdef GMX_MPI
 240     if (n > 0)
 241     {
 242         if (flags & PP_PME_CHARGE)
 243         {
 244             MPI_Isend(chargeA, n*sizeof(real), MPI_BYTE,
 245                       dd->pme_nodeid, eCommType_ChargeA, cr->mpi_comm_mysim,
 246                       &dd->req_pme[dd->nreq_pme++]);
 247         }
 248         if (flags & PP_PME_CHARGEB)
 249         {
 250             MPI_Isend(chargeB, n*sizeof(real), MPI_BYTE,
 251                       dd->pme_nodeid, eCommType_ChargeB, cr->mpi_comm_mysim,
 252                       &dd->req_pme[dd->nreq_pme++]);
 253         }
 254         if (flags & PP_PME_C6)
 255         {
 256             MPI_Isend(c6A, n*sizeof(real), MPI_BYTE,
 257                       dd->pme_nodeid, eCommType_C6A, cr->mpi_comm_mysim,
 258                       &dd->req_pme[dd->nreq_pme++]);
 259         }
 260         if (flags & PP_PME_C6B)
 261         {
 262             MPI_Isend(c6B, n*sizeof(real), MPI_BYTE,
 263                       dd->pme_nodeid, eCommType_C6B, cr->mpi_comm_mysim,
 264                       &dd->req_pme[dd->nreq_pme++]);
 265         }
 266         if (flags & PP_PME_SIGMA)
 267         {
 268             MPI_Isend(sigmaA, n*sizeof(real), MPI_BYTE,
 269                       dd->pme_nodeid, eCommType_SigmaA, cr->mpi_comm_mysim,
 270                       &dd->req_pme[dd->nreq_pme++]);
 271         }
 272         if (flags & PP_PME_SIGMAB)
 273         {
 274             MPI_Isend(sigmaB, n*sizeof(real), MPI_BYTE,
 275                       dd->pme_nodeid, eCommType_SigmaB, cr->mpi_comm_mysim,
 276                       &dd->req_pme[dd->nreq_pme++]);
 277         }
 278         if (flags & PP_PME_COORD)
 279         {
 280             MPI_Isend(x[0], n*sizeof(rvec), MPI_BYTE,
 281                       dd->pme_nodeid, eCommType_COORD, cr->mpi_comm_mysim,
 282                       &dd->req_pme[dd->nreq_pme++]);
 283         }
 284     }
 285
 286 #ifndef GMX_PME_DELAYED_WAIT
 287     /* Wait for the data to arrive */
 288     /* We can skip this wait as we are sure x and q will not be modified
 289      * before the next call to gmx_pme_send_x_q or gmx_pme_receive_f.
 290      */
 291     gmx_pme_send_params_coords_wait(dd);
 292 #endif
 293 #endif
 294 }
 295
 296 void gmx_pme_send_parameters(t_commrec *cr,
 297                              gmx_bool bFreeEnergy_q, gmx_bool bFreeEnergy_lj,
 298                              real *chargeA, real *chargeB,
 299                              real *c6A, real *c6B, real *sigmaA, real *sigmaB,
 300                              int maxshift_x, int maxshift_y)
 301 {
 302     int flags;
 303
 304     flags = PP_PME_CHARGE | PP_PME_C6 | PP_PME_SIGMA;
 305     if (bFreeEnergy_q || bFreeEnergy_lj)
 306     {
 307         /* Assumes that the B state flags are in the bits just above
 308          * the ones for the A state. */
 309         flags |= (flags << 1);
 310     }
 311
 312     gmx_pme_send_params_coords(cr, flags, chargeA, chargeB, c6A, c6B, sigmaA, sigmaB,
 313                                NULL, NULL, 0, 0, maxshift_x, maxshift_y, -1);
 314 }
 315
 316 void gmx_pme_send_coordinates(t_commrec *cr, matrix box, rvec *x,
 317                               gmx_bool bFreeEnergy_q, gmx_bool bFreeEnergy_lj,
 318                               real lambda_q, real lambda_lj,
 319                               gmx_bool bEnerVir, int pme_flags,
 320                               gmx_large_int_t step)
 321 {
 322     int flags;
 323
 324     flags = pme_flags | PP_PME_COORD;
 325     if (bFreeEnergy_q)
 326     {
 327         flags |= PP_PME_FEP_Q;
 328     }
 329     if (bFreeEnergy_lj)
 330     {
 331         flags |= PP_PME_FEP_LJ;
 332     }
 333     if (bEnerVir)
 334     {
 335         flags |= PP_PME_ENER_VIR;
 336     }
 337     gmx_pme_send_params_coords(cr, flags, NULL, NULL, NULL, NULL, NULL, NULL,
 338                                box, x, lambda_q, lambda_lj, 0, 0, step);
 339 }
 340
 341 void gmx_pme_send_finish(t_commrec *cr)
 342 {
 343     int flags;
 344
 345     flags = PP_PME_FINISH;
 346
 347     gmx_pme_send_params_coords(cr, flags, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0, -1);
 348 }
 349
 350 void gmx_pme_send_switchgrid(t_commrec gmx_unused *cr,
 351                              ivec gmx_unused       grid_size,
 352                              real gmx_unused       ewaldcoeff_q,
 353                              real gmx_unused       ewaldcoeff_lj)
 354 {
 355 #ifdef GMX_MPI
 356     gmx_pme_comm_n_box_t cnb;
 357
 358     /* Only let one PP node signal each PME node */
 359     if (cr->dd->pme_receive_vir_ener)
 360     {
 361         cnb.flags = PP_PME_SWITCHGRID;
 362         copy_ivec(grid_size, cnb.grid_size);
 363         cnb.ewaldcoeff_q  = ewaldcoeff_q;
 364         cnb.ewaldcoeff_lj = ewaldcoeff_lj;
 365
 366         /* We send this, uncommon, message blocking to simplify the code */
 367         MPI_Send(&cnb, sizeof(cnb), MPI_BYTE,
 368                  cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
 369     }
 370 #endif
 371 }
 372
 373 void gmx_pme_send_resetcounters(t_commrec gmx_unused *cr, gmx_large_int_t gmx_unused step)
 374 {
 375 #ifdef GMX_MPI
 376     gmx_pme_comm_n_box_t cnb;
 377
 378     /* Only let one PP node signal each PME node */
 379     if (cr->dd->pme_receive_vir_ener)
 380     {
 381         cnb.flags = PP_PME_RESETCOUNTERS;
 382         cnb.step  = step;
 383
 384         /* We send this, uncommon, message blocking to simplify the code */
 385         MPI_Send(&cnb, sizeof(cnb), MPI_BYTE,
 386                  cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
 387     }
 388 #endif
 389 }
 390
 391 int gmx_pme_recv_params_coords(struct gmx_pme_pp          *pme_pp,
 392                                int                        *natoms,
 393                                real                      **chargeA,
 394                                real                      **chargeB,
 395                                real                      **c6A,
 396                                real                      **c6B,
 397                                real                      **sigmaA,
 398                                real                      **sigmaB,
 399                                matrix gmx_unused           box,
 400                                rvec                      **x,
 401                                rvec                      **f,
 402                                int gmx_unused             *maxshift_x,
 403                                int gmx_unused             *maxshift_y,
 404                                gmx_bool gmx_unused        *bFreeEnergy_q,
 405                                gmx_bool gmx_unused        *bFreeEnergy_lj,
 406                                real gmx_unused            *lambda_q,
 407                                real gmx_unused            *lambda_lj,
 408                                gmx_bool gmx_unused        *bEnerVir,
 409                                int                        *pme_flags,
 410                                gmx_large_int_t gmx_unused *step,
 411                                ivec gmx_unused             grid_size,
 412                                real gmx_unused            *ewaldcoeff_q,
 413                                real gmx_unused            *ewaldcoeff_lj)
 414 {
 415     gmx_pme_comm_n_box_t cnb;
 416     int                  nat = 0, q, messages, sender;
 417     real                *charge_pp;
 418
 419     messages = 0;
 420
 421     /* avoid compiler warning about unused variable without MPI support */
 422     cnb.flags  = 0;
 423     *pme_flags = 0;
 424 #ifdef GMX_MPI
 425     do
 426     {
 427         /* Receive the send count, box and time step from the peer PP node */
 428         MPI_Recv(&cnb, sizeof(cnb), MPI_BYTE,
 429                  pme_pp->node_peer, eCommType_CNB,
 430                  pme_pp->mpi_comm_mysim, MPI_STATUS_IGNORE);
 431
 432         if (debug)
 433         {
 434             fprintf(debug, "PME only node receiving:%s%s%s%s%s\n",
 435                     (cnb.flags & PP_PME_CHARGE)        ? " charges" : "",
 436                     (cnb.flags & PP_PME_COORD )        ? " coordinates" : "",
 437                     (cnb.flags & PP_PME_FINISH)        ? " finish" : "",
 438                     (cnb.flags & PP_PME_SWITCHGRID)    ? " switch grid" : "",
 439                     (cnb.flags & PP_PME_RESETCOUNTERS) ? " reset counters" : "");
 440         }
 441
 442         if (cnb.flags & PP_PME_SWITCHGRID)
 443         {
 444             /* Special case, receive the new parameters and return */
 445             copy_ivec(cnb.grid_size, grid_size);
 446             *ewaldcoeff_q  = cnb.ewaldcoeff_q;
 447             *ewaldcoeff_lj = cnb.ewaldcoeff_lj;
 448             return pmerecvqxSWITCHGRID;
 449         }
 450
 451         if (cnb.flags & PP_PME_RESETCOUNTERS)
 452         {
 453             /* Special case, receive the step and return */
 454             *step = cnb.step;
 455
 456             return pmerecvqxRESETCOUNTERS;
 457         }
 458
 459         if (cnb.flags & (PP_PME_CHARGE | PP_PME_C6 | PP_PME_SIGMA))
 460         {
 461             /* Receive the send counts from the other PP nodes */
 462             for (sender = 0; sender < pme_pp->nnode; sender++)
 463             {
 464                 if (pme_pp->node[sender] == pme_pp->node_peer)
 465                 {
 466                     pme_pp->nat[sender] = cnb.natoms;
 467                 }
 468                 else
 469                 {
 470                     MPI_Irecv(&(pme_pp->nat[sender]), sizeof(pme_pp->nat[0]),
 471                               MPI_BYTE,
 472                               pme_pp->node[sender], eCommType_CNB,
 473                               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 474                 }
 475             }
 476             MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 477             messages = 0;
 478
 479             nat = 0;
 480             for (sender = 0; sender < pme_pp->nnode; sender++)
 481             {
 482                 nat += pme_pp->nat[sender];
 483             }
 484
 485             if (nat > pme_pp->nalloc)
 486             {
 487                 pme_pp->nalloc = over_alloc_dd(nat);
 488                 if (cnb.flags & PP_PME_CHARGE)
 489                 {
 490                     srenew(pme_pp->chargeA, pme_pp->nalloc);
 491                 }
 492                 if (cnb.flags & PP_PME_CHARGEB)
 493                 {
 494                     srenew(pme_pp->chargeB, pme_pp->nalloc);
 495                 }
 496                 if (cnb.flags & PP_PME_C6)
 497                 {
 498                     srenew(pme_pp->c6A, pme_pp->nalloc);
 499                 }
 500                 if (cnb.flags & PP_PME_C6B)
 501                 {
 502                     srenew(pme_pp->c6B, pme_pp->nalloc);
 503                 }
 504                 if (cnb.flags & PP_PME_SIGMA)
 505                 {
 506                     srenew(pme_pp->sigmaA, pme_pp->nalloc);
 507                 }
 508                 if (cnb.flags & PP_PME_SIGMAB)
 509                 {
 510                     srenew(pme_pp->sigmaB, pme_pp->nalloc);
 511                 }
 512                 srenew(pme_pp->x, pme_pp->nalloc);
 513                 srenew(pme_pp->f, pme_pp->nalloc);
 514             }
 515
 516             /* maxshift is sent when the charges are sent */
 517             *maxshift_x = cnb.maxshift_x;
 518             *maxshift_y = cnb.maxshift_y;
 519
 520             /* Receive the charges in place */
 521             for (q = 0; q < eCommType_NR; q++)
 522             {
 523                 if (!(cnb.flags & (PP_PME_CHARGE<<q)))
 524                 {
 525                     continue;
 526                 }
 527                 switch (q)
 528                 {
 529                     case eCommType_ChargeA: charge_pp = pme_pp->chargeA; break;
 530                     case eCommType_ChargeB: charge_pp = pme_pp->chargeB; break;
 531                     case eCommType_C6A:     charge_pp = pme_pp->c6A;     break;
 532                     case eCommType_C6B:     charge_pp = pme_pp->c6B;     break;
 533                     case eCommType_SigmaA:  charge_pp = pme_pp->sigmaA;  break;
 534                     case eCommType_SigmaB:  charge_pp = pme_pp->sigmaB;  break;
 535                     default: gmx_incons("Wrong eCommType");
 536                 }
 537                 nat = 0;
 538                 for (sender = 0; sender < pme_pp->nnode; sender++)
 539                 {
 540                     if (pme_pp->nat[sender] > 0)
 541                     {
 542                         MPI_Irecv(charge_pp+nat,
 543                                   pme_pp->nat[sender]*sizeof(real),
 544                                   MPI_BYTE,
 545                                   pme_pp->node[sender], q,
 546                                   pme_pp->mpi_comm_mysim,
 547                                   &pme_pp->req[messages++]);
 548                         nat += pme_pp->nat[sender];
 549                         if (debug)
 550                         {
 551                             fprintf(debug, "Received from PP node %d: %d "
 552                                     "charges\n",
 553                                     pme_pp->node[sender], pme_pp->nat[sender]);
 554                         }
 555                     }
 556                 }
 557             }
 558
 559             pme_pp->flags_charge = cnb.flags;
 560         }
 561
 562         if (cnb.flags & PP_PME_COORD)
 563         {
 564             if (!(pme_pp->flags_charge & (PP_PME_CHARGE | PP_PME_C6)))
 565             {
 566                 gmx_incons("PME-only node received coordinates before charges and/or C6-values"
 567                            );
 568             }
 569
 570             /* The box, FE flag and lambda are sent along with the coordinates
 571              *  */
 572             copy_mat(cnb.box, box);
 573             *bFreeEnergy_q  = (cnb.flags & PP_PME_FEP_Q);
 574             *bFreeEnergy_lj = (cnb.flags & PP_PME_FEP_LJ);
 575             *lambda_q       = cnb.lambda_q;
 576             *lambda_lj      = cnb.lambda_lj;
 577             *bEnerVir       = (cnb.flags & PP_PME_ENER_VIR);
 578             *pme_flags      = cnb.flags;
 579
 580             if (*bFreeEnergy_q && !(pme_pp->flags_charge & PP_PME_CHARGEB))
 581             {
 582                 gmx_incons("PME-only node received free energy request, but "
 583                            "did not receive B-state charges");
 584             }
 585
 586             if (*bFreeEnergy_lj && !(pme_pp->flags_charge & PP_PME_C6B))
 587             {
 588                 gmx_incons("PME-only node received free energy request, but "
 589                            "did not receive B-state C6-values");
 590             }
 591
 592             /* Receive the coordinates in place */
 593             nat = 0;
 594             for (sender = 0; sender < pme_pp->nnode; sender++)
 595             {
 596                 if (pme_pp->nat[sender] > 0)
 597                 {
 598                     MPI_Irecv(pme_pp->x[nat], pme_pp->nat[sender]*sizeof(rvec),
 599                               MPI_BYTE,
 600                               pme_pp->node[sender], eCommType_COORD,
 601                               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 602                     nat += pme_pp->nat[sender];
 603                     if (debug)
 604                     {
 605                         fprintf(debug, "Received from PP node %d: %d "
 606                                 "coordinates\n",
 607                                 pme_pp->node[sender], pme_pp->nat[sender]);
 608                     }
 609                 }
 610             }
 611         }
 612
 613         /* Wait for the coordinates and/or charges to arrive */
 614         MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 615         messages = 0;
 616     }
 617     while (!(cnb.flags & (PP_PME_COORD | PP_PME_FINISH)));
 618
 619     *step = cnb.step;
 620 #endif
 621
 622     *natoms  = nat;
 623     *chargeA = pme_pp->chargeA;
 624     *chargeB = pme_pp->chargeB;
 625     *c6A     = pme_pp->c6A;
 626     *c6B     = pme_pp->c6B;
 627     *sigmaA  = pme_pp->sigmaA;
 628     *sigmaB  = pme_pp->sigmaB;
 629     *x       = pme_pp->x;
 630     *f       = pme_pp->f;
 631
 632     return ((cnb.flags & PP_PME_FINISH) ? pmerecvqxFINISH : pmerecvqxX);
 633 }
 634
 635 static void receive_virial_energy(t_commrec *cr,
 636                                   matrix vir_q, real *energy_q,
 637                                   matrix vir_lj, real *energy_lj,
 638                                   real *dvdlambda_q, real *dvdlambda_lj,
 639                                   float *pme_cycles)
 640 {
 641     gmx_pme_comm_vir_ene_t cve;
 642
 643     if (cr->dd->pme_receive_vir_ener)
 644     {
 645         if (debug)
 646         {
 647             fprintf(debug,
 648                     "PP node %d receiving from PME node %d: virial and energy\n",
 649                     cr->sim_nodeid, cr->dd->pme_nodeid);
 650         }
 651 #ifdef GMX_MPI
 652         MPI_Recv(&cve, sizeof(cve), MPI_BYTE, cr->dd->pme_nodeid, 1, cr->mpi_comm_mysim,
 653                  MPI_STATUS_IGNORE);
 654 #else
 655         memset(&cve, 0, sizeof(cve));
 656 #endif
 657
 658         m_add(vir_q, cve.vir_q, vir_q);
 659         m_add(vir_lj, cve.vir_lj, vir_lj);
 660         *energy_q      = cve.energy_q;
 661         *energy_lj     = cve.energy_lj;
 662         *dvdlambda_q  += cve.dvdlambda_q;
 663         *dvdlambda_lj += cve.dvdlambda_lj;
 664         *pme_cycles    = cve.cycles;
 665
 666         if (cve.stop_cond != gmx_stop_cond_none)
 667         {
 668             gmx_set_stop_condition(cve.stop_cond);
 669         }
 670     }
 671     else
 672     {
 673         *energy_q   = 0;
 674         *energy_lj  = 0;
 675         *pme_cycles = 0;
 676     }
 677 }
 678
 679 void gmx_pme_receive_f(t_commrec *cr,
 680                        rvec f[], matrix vir_q, real *energy_q,
 681                        matrix vir_lj, real *energy_lj,
 682                        real *dvdlambda_q, real *dvdlambda_lj,
 683                        float *pme_cycles)
 684 {
 685     int natoms, i;
 686
 687 #ifdef GMX_PME_DELAYED_WAIT
 688     /* Wait for the x request to finish */
 689     gmx_pme_send_params_coords_wait(cr->dd);
 690 #endif
 691
 692     natoms = cr->dd->nat_home;
 693
 694     if (natoms > cr->dd->pme_recv_f_alloc)
 695     {
 696         cr->dd->pme_recv_f_alloc = over_alloc_dd(natoms);
 697         srenew(cr->dd->pme_recv_f_buf, cr->dd->pme_recv_f_alloc);
 698     }
 699
 700 #ifdef GMX_MPI
 701     MPI_Recv(cr->dd->pme_recv_f_buf[0],
 702              natoms*sizeof(rvec), MPI_BYTE,
 703              cr->dd->pme_nodeid, 0, cr->mpi_comm_mysim,
 704              MPI_STATUS_IGNORE);
 705 #endif
 706
 707     for (i = 0; i < natoms; i++)
 708     {
 709         rvec_inc(f[i], cr->dd->pme_recv_f_buf[i]);
 710     }
 711
 712
 713     receive_virial_energy(cr, vir_q, energy_q, vir_lj, energy_lj, dvdlambda_q, dvdlambda_lj, pme_cycles);
 714 }
 715
 716 void gmx_pme_send_force_vir_ener(struct gmx_pme_pp *pme_pp,
 717                                  rvec gmx_unused *f,
 718                                  matrix vir_q, real energy_q,
 719                                  matrix vir_lj, real energy_lj,
 720                                  real dvdlambda_q, real dvdlambda_lj,
 721                                  float cycles)
 722 {
 723     gmx_pme_comm_vir_ene_t cve;
 724     int                    messages, ind_start, ind_end, receiver;
 725
 726     cve.cycles = cycles;
 727
 728     /* Now the evaluated forces have to be transferred to the PP nodes */
 729     messages = 0;
 730     ind_end  = 0;
 731     for (receiver = 0; receiver < pme_pp->nnode; receiver++)
 732     {
 733         ind_start = ind_end;
 734         ind_end   = ind_start + pme_pp->nat[receiver];
 735 #ifdef GMX_MPI
 736         if (MPI_Isend(f[ind_start], (ind_end-ind_start)*sizeof(rvec), MPI_BYTE,
 737                       pme_pp->node[receiver], 0,
 738                       pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]) != 0)
 739         {
 740             gmx_comm("MPI_Isend failed in do_pmeonly");
 741         }
 742 #endif
 743     }
 744
 745     /* send virial and energy to our last PP node */
 746     copy_mat(vir_q, cve.vir_q);
 747     copy_mat(vir_lj, cve.vir_lj);
 748     cve.energy_q     = energy_q;
 749     cve.energy_lj    = energy_lj;
 750     cve.dvdlambda_q  = dvdlambda_q;
 751     cve.dvdlambda_lj = dvdlambda_lj;
 752     /* check for the signals to send back to a PP node */
 753     cve.stop_cond = gmx_get_stop_condition();
 754
 755     cve.cycles = cycles;
 756
 757     if (debug)
 758     {
 759         fprintf(debug, "PME node sending to PP node %d: virial and energy\n",
 760                 pme_pp->node_peer);
 761     }
 762 #ifdef GMX_MPI
 763     MPI_Isend(&cve, sizeof(cve), MPI_BYTE,
 764               pme_pp->node_peer, 1,
 765               pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
 766
 767     /* Wait for the forces to arrive */
 768     MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
 769 #endif
 770 }