2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * \brief Tests for the halo exchange
38 * The test sets up a 2D rank topology and performs a coordinate halo
39 * exchange (using the pre-existing CPU codepath), with 2 pulses in
40 * the first dimension and 1 pulse in the second. Each pulse involves
41 * a few non-contiguous indices. The sending rank, atom number and
42 * spatial 3D index are encoded in the x values, to allow correctness
43 * checking following the halo exchange.
45 * \todo Add more test variations
46 * \todo Port to GPU codepath
48 * \author Alan Gray <alang@nvidia.com>
49 * \ingroup module_domdec
56 #include <gtest/gtest.h>
58 #include "gromacs/domdec/atomdistribution.h"
59 #include "gromacs/domdec/domdec_internal.h"
60 #include "gromacs/domdec/gpuhaloexchange.h"
61 #include "gromacs/mdtypes/inputrec.h"
63 #include "testutils/mpitest.h"
70 /*! \brief Get encoded numerical value for sending rank, atom number and spatial 3D index
72 * \param [in] sendRank MPI rank of sender
73 * \param [in] atomNumber Atom number
74 * \param [in] spatial3dIndex Spatial 3D Index
76 * \returns Encoded value
78 float encodedValue(const int sendRank, const int atomNumber, const int spatial3dIndex)
80 return sendRank * 1000 + atomNumber * 100 + spatial3dIndex;
83 /*! \brief Initialize halo array
85 * \param [in] x Atom coordinate data array
86 * \param [in] numHomeAtoms Number of home atoms
87 * \param [in] numAtomsTotal Total number of atoms, including halo
89 void initHaloData(RVec* x, const int numHomeAtoms, const int numAtomsTotal)
92 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
94 for (int i = 0; i < numAtomsTotal; i++)
96 for (int j = 0; j < DIM; j++)
98 x[i][j] = i < numHomeAtoms ? encodedValue(rank, i, j) : -1;
103 /*! \brief Define 2D rank topology with 4 MPI tasks
110 * \param [in] dd Domain decomposition object
112 void define2dRankTopology(gmx_domdec_t* dd)
116 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
121 dd->neighbor[0][0] = 1;
122 dd->neighbor[0][1] = 1;
123 dd->neighbor[1][0] = 2;
124 dd->neighbor[1][1] = 2;
127 dd->neighbor[0][0] = 0;
128 dd->neighbor[0][1] = 0;
129 dd->neighbor[1][0] = 3;
130 dd->neighbor[1][1] = 3;
133 dd->neighbor[0][0] = 3;
134 dd->neighbor[0][1] = 3;
135 dd->neighbor[1][0] = 0;
136 dd->neighbor[1][1] = 0;
139 dd->neighbor[0][0] = 2;
140 dd->neighbor[0][1] = 2;
141 dd->neighbor[1][0] = 1;
142 dd->neighbor[1][1] = 1;
147 /*! \brief Define a 2D halo with 2 pulses in the first dimension
149 * \param [in] dd Domain decomposition object
150 * \param [in] indvec Vector of index vectors
152 void define2dHaloWith2PulsesInDim1(gmx_domdec_t* dd, std::vector<gmx_domdec_ind_t> indvec)
156 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
158 std::vector<int> indexvec;
159 gmx_domdec_ind_t ind;
163 for (int dimIndex = 0; dimIndex < dd->ndim; dimIndex++)
166 // Set up indices involved in halo
170 dd->comm->cd[dimIndex].receiveInPlace = true;
171 dd->dim[dimIndex] = 0;
172 dd->ci[dimIndex] = rank;
174 // First pulse involves (arbitrary) indices 1 and 3
175 indexvec.push_back(1);
176 indexvec.push_back(3);
178 ind.index = indexvec;
179 ind.nsend[nzone + 1] = 2;
180 ind.nrecv[nzone + 1] = 2;
181 indvec.push_back(ind);
183 if (dimIndex == 0) // Add another pulse with (arbitrary) indices 4,5,7
187 dd->comm->cd[dimIndex].ind = indvec;
189 indexvec.push_back(4);
190 indexvec.push_back(5);
191 indexvec.push_back(7);
193 ind.index = indexvec;
194 ind.nsend[nzone + 1] = 3;
195 ind.nrecv[nzone + 1] = 3;
196 indvec.push_back(ind);
199 dd->comm->cd[dimIndex].ind = indvec;
205 /*! \brief Check results for above-defined 2D halo with 2 pulses in the first dimension
207 * \param [in] x Atom coordinate data array
208 * \param [in] dd Domain decomposition object
209 * \param [in] numHomeAtoms Number of home atoms
211 void checkResults2dHaloWith2PulsesInDim1(const RVec* x, const gmx_domdec_t* dd, const int numHomeAtoms)
213 // Check results are expected from values encoded in x data
214 for (int j = 0; j < DIM; j++)
216 // First Pulse in first dim: atoms 1 and 3 from forward horizontal neighbour
217 EXPECT_EQ(x[numHomeAtoms][j], encodedValue(dd->neighbor[0][0], 1, j));
218 EXPECT_EQ(x[numHomeAtoms + 1][j], encodedValue(dd->neighbor[0][0], 3, j));
219 // Second Pulse in first dim: atoms 4,5,7 from forward horizontal neighbour
220 EXPECT_EQ(x[numHomeAtoms + 2][j], encodedValue(dd->neighbor[0][0], 4, j));
221 EXPECT_EQ(x[numHomeAtoms + 3][j], encodedValue(dd->neighbor[0][0], 5, j));
222 EXPECT_EQ(x[numHomeAtoms + 4][j], encodedValue(dd->neighbor[0][0], 7, j));
223 // First Pulse in second dim: atoms 1 and 3 from forward vertical neighbour
224 EXPECT_EQ(x[numHomeAtoms + 5][j], encodedValue(dd->neighbor[1][0], 1, j));
225 EXPECT_EQ(x[numHomeAtoms + 6][j], encodedValue(dd->neighbor[1][0], 3, j));
230 TEST(HaloExchangeTest, Coordinates2dHaloWith2PulsesInDim1)
235 const int numHomeAtoms = 10;
236 const int numHaloAtoms = 7;
237 const int numAtomsTotal = numHomeAtoms + numHaloAtoms;
238 RVec x[numAtomsTotal];
239 initHaloData(x, numHomeAtoms, numAtomsTotal);
244 dd.mpi_comm_all = MPI_COMM_WORLD;
245 gmx_domdec_comm_t comm;
247 dd.unitCellInfo.haveScrewPBC = false;
249 DDAtomRanges atomRanges;
250 atomRanges.setEnd(DDAtomRanges::Type::Home, numHomeAtoms);
251 dd.comm->atomRanges = atomRanges;
253 define2dRankTopology(&dd);
255 std::vector<gmx_domdec_ind_t> indvec;
256 define2dHaloWith2PulsesInDim1(&dd, indvec);
258 // Perform halo exchange
259 matrix box = { { 0., 0., 0. } };
260 dd_move_x(&dd, box, static_cast<ArrayRef<RVec>>(x), nullptr);
263 checkResults2dHaloWith2PulsesInDim1(x, &dd, numHomeAtoms);