2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team.
6 * Copyright (c) 2013,2014,2015,2017, by the GROMACS development team, led by
7 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
8 * and including many others, as listed in the AUTHORS file in the
9 * top-level source directory and at http://www.gromacs.org.
11 * GROMACS is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * GROMACS is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with GROMACS; if not, see
23 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
24 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 * If you want to redistribute modifications to GROMACS, please
27 * consider that scientific software is very special. Version
28 * control is crucial - bugs must be traceable. We will be happy to
29 * consider code for inclusion in the official distribution, but
30 * derived work must not be called official GROMACS. Details are found
31 * in the README & COPYING files - if they are missing, get the
32 * official version at http://www.gromacs.org.
34 * To help us fund GROMACS development, we humbly ask that you cite
35 * the research papers on the package. Check out http://www.gromacs.org.
37 #ifndef GMX_MDTYPES_COMMREC_H
38 #define GMX_MDTYPES_COMMREC_H
42 #include "gromacs/math/vectypes.h"
43 #include "gromacs/utility/basedefinitions.h"
44 #include "gromacs/utility/gmxmpi.h"
45 #include "gromacs/utility/real.h"
50 /* these buffers are used as destination buffers if MPI_IN_PLACE isn't
52 int *ibuf; /* for ints */
58 float *fbuf; /* for floats */
61 double *dbuf; /* for doubles */
65 struct gmx_multisim_t {
68 MPI_Group mpi_group_masters;
69 MPI_Comm mpi_comm_masters;
70 /* these buffers are used as destination buffers if MPI_IN_PLACE isn't
72 mpi_in_place_buf_t *mpb;
75 #define DUTY_PP (1<<0)
76 #define DUTY_PME (1<<1)
87 /* The nodeids in one sim are numbered sequentially from 0.
88 * All communication within some simulation should happen
89 * in mpi_comm_mysim, or its subset mpi_comm_mygroup.
91 int sim_nodeid, nnodes, npmenodes;
94 /* Not used yet: int threadid, nthreads; */
95 /* The nodeid in the PP/PME, PP or PME group */
97 MPI_Comm mpi_comm_mysim;
98 MPI_Comm mpi_comm_mygroup;
100 /* MPI ranks and a communicator within a physical node for hardware access */
101 MPI_Comm mpi_comm_physicalnode; /* communicator for all ranks of the physical node
102 * NOTE: this communicator should only be used during initialization and finalization, as it can contain ranks from PP, PME and multiple simulations with multisim
104 int nrank_intranode; /* nr of ranks on this physical node */
105 int rank_intranode; /* our rank on this physical node */
106 int nrank_pp_intranode; /* as nrank_intranode, for particle-particle only */
107 int rank_pp_intranode; /* as rank_intranode, for particle-particle only */
111 /* For domain decomposition */
114 /* The duties of this node, see the defines above */
119 /* these buffers are used as destination buffers if MPI_IN_PLACE isn't
121 mpi_in_place_buf_t *mpb;
124 //! True if this is a simulation with more than 1 node
125 #define PAR(cr) ((cr)->nnodes > 1)
127 //! True of this is the master node
128 #define MASTER(cr) (((cr)->nodeid == 0) || !PAR(cr))
130 //! True if this is the particle-particle master
131 #define SIMMASTER(cr) ((MASTER(cr) && ((cr)->duty & DUTY_PP)) || !PAR(cr))
133 //! The node id for this rank
134 #define RANK(cr, nodeid) (nodeid)
136 //! The node id for the master
137 #define MASTERRANK(cr) (0)
139 /*! \brief Do we use domain decomposition
141 * Note that even with particle decomposition removed, the use of
142 * non-DD parallelization in TPI, NM and multi-simulations means that
143 * PAR(cr) and DOMAINDECOMP(cr) are not universally synonymous. In
144 * particular, DOMAINDECOMP(cr) == true indicates that there is more
145 * than one domain, not just that the dd algorithm is active. */
146 #define DOMAINDECOMP(cr) (((cr)->dd != NULL) && PAR(cr))
148 //! Are we doing multiple independent simulations
149 #define MULTISIM(cr) ((cr)->ms)
151 //! Are we the master node of a multisimulation
152 #define MASTERSIM(ms) ((ms)->sim == 0)
154 //! The master of all (the node that prints the remaining run time etc.)
155 #define MULTIMASTER(cr) (SIMMASTER(cr) && (!MULTISIM(cr) || MASTERSIM((cr)->ms)))