2 This source code file is part of thread_mpi.
3 Written by Sander Pronk, Erik Lindahl, and possibly others.
5 Copyright (c) 2009, Sander Pronk, Erik Lindahl.
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10 1) Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
12 2) Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15 3) Neither the name of the copyright holders nor the
16 names of its contributors may be used to endorse or promote products
17 derived from this software without specific prior written permission.
19 THIS SOFTWARE IS PROVIDED BY US ''AS IS'' AND ANY
20 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL WE BE LIABLE FOR ANY
23 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 If you want to redistribute modifications, please consider that
31 scientific software is very special. Version control is crucial -
32 bugs must be traceable. We will be happy to consider code for
33 inclusion in the official distribution, but derived work should not
34 be called official thread_mpi. Details are found in the README & COPYING
38 #ifdef HAVE_TMPI_CONFIG_H
39 #include "tmpi_config.h"
61 /* topology functions */
62 int tMPI_Topo_test(tMPI_Comm comm, int *status)
65 tMPI_Trace_print("tMPI_Topo_test(%p, %p)", comm, status);
70 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
77 /*else if (comm->graph)
81 *status = TMPI_UNDEFINED;
87 int tMPI_Cartdim_get(tMPI_Comm comm, int *ndims)
90 tMPI_Trace_print("tMPI_Cartdim_get(%p, %p)", comm, ndims);
94 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
96 if (!comm->cart || comm->cart->ndims == 0)
100 *ndims = comm->cart->ndims;
105 int tMPI_Cart_get(tMPI_Comm comm, int maxdims, int *dims, int *periods,
109 int myrank = tMPI_Comm_seek_rank(comm, tMPI_Get_current());
112 tMPI_Trace_print("tMPI_Cart_get(%p, %d, %p, %p, %p)", comm, maxdims,
113 dims, periods, coords);
117 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
119 if (!comm->cart || comm->cart->ndims == 0)
124 tMPI_Cart_coords(comm, myrank, maxdims, coords);
126 for (i = 0; i < comm->cart->ndims; i++)
130 return tMPI_Error(comm, TMPI_ERR_DIMS);
132 dims[i] = comm->cart->dims[i];
133 periods[i] = comm->cart->periods[i];
139 int tMPI_Cart_rank(tMPI_Comm comm, int *coords, int *rank)
141 int i, mul = 1, ret = 0;
144 tMPI_Trace_print("tMPI_Cart_get(%p, %p, %p)", comm, coords, rank);
148 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
150 if (!comm->cart || comm->cart->ndims == 0)
155 /* because of row-major ordering, we count the dimensions down */
156 for (i = comm->cart->ndims-1; i >= 0; i--)
158 int rcoord = coords[i];
159 if (comm->cart->periods[i])
161 /* apply periodic boundary conditions */
162 rcoord = rcoord % comm->cart->dims[i];
165 rcoord += comm->cart->dims[i];
170 if (rcoord < 0 || rcoord >= comm->cart->dims[i])
172 return tMPI_Error(comm, TMPI_ERR_DIMS);
176 mul *= comm->cart->dims[i];
182 int tMPI_Cart_coords(tMPI_Comm comm, int rank, int maxdims, int *coords)
185 int rank_left = rank;
188 tMPI_Trace_print("tMPI_Cart_coords(%p, %d, %d, %p)", comm, rank, maxdims,
193 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
195 if (!comm->cart || comm->cart->ndims == 0)
199 if (maxdims < comm->cart->ndims)
201 return tMPI_Error(comm, TMPI_ERR_DIMS);
204 /* again, row-major ordering */
205 for (i = comm->cart->ndims-1; i >= 0; i--)
207 coords[i] = rank_left%comm->cart->dims[i];
208 rank_left /= comm->cart->dims[i];
216 int tMPI_Cart_map(tMPI_Comm comm, int ndims, int *dims, int *periods,
219 /* this function doesn't actually do anything beyond returning the current
220 rank (or TMPI_UNDEFINED if it doesn't fit in the new topology */
221 int myrank = tMPI_Comm_seek_rank(comm, tMPI_Get_current());
226 tMPI_Trace_print("tMPI_Cart_map(%p, %d, %p, %p, %p)", comm, ndims, dims,
231 return tMPI_Error(TMPI_COMM_WORLD, TMPI_ERR_COMM);
235 return tMPI_Error(comm, TMPI_ERR_DIMS);
238 /* calculate the total number of procs in cartesian comm */
239 for (i = 0; i < ndims; i++)
246 *newrank = TMPI_UNDEFINED;
257 /* initialize Cartesian topology info in comm. If ndims==0, dims and periods
258 are not referenced */
259 static void tMPI_Cart_init(tMPI_Comm *comm_cart, int ndims, int *dims,
267 tMPI_Comm_rank(*comm_cart, &newrank);
272 (*comm_cart)->cart = (struct cart_topol*)tMPI_Malloc(
273 sizeof(struct cart_topol));
274 (*comm_cart)->cart->dims = (int*)tMPI_Malloc(ndims*sizeof(int));
275 (*comm_cart)->cart->periods = (int*)tMPI_Malloc(ndims*sizeof(int));
276 (*comm_cart)->cart->ndims = ndims;
277 for (i = 0; i < ndims; i++)
279 (*comm_cart)->cart->dims[i] = dims[i];
280 (*comm_cart)->cart->periods[i] = periods[i];
284 /* and we add a barrier to make sure the cart object is seen by
285 every thread that is part of the new communicator */
288 tMPI_Barrier_wait( &( (*comm_cart)->barrier) );
292 void tMPI_Cart_destroy(struct cart_topol *cart)
301 int tMPI_Cart_create(tMPI_Comm comm_old, int ndims, int *dims, int *periods,
302 int reorder, tMPI_Comm *comm_cart)
304 int myrank = tMPI_Comm_seek_rank(comm_old, tMPI_Get_current());
312 tMPI_Trace_print("tMPI_Cart_create(%p, %d, %p, %p, %d, %p)", comm_old,
313 ndims, dims, periods, reorder, comm_cart);
317 return tMPI_Error(comm_old, TMPI_ERR_COMM);
319 /* calculate the total number of procs in cartesian comm */
320 for (i = 0; i < ndims; i++)
324 /* refuse to create if there's not enough procs */
325 if (comm_old->grp.N < Ntot)
327 *comm_cart = TMPI_COMM_NULL;
329 return tMPI_Error(comm_old, TMPI_ERR_CART_CREATE_NPROCS);
335 key = TMPI_UNDEFINED;
340 tMPI_Cart_map(comm_old, ndims, dims, periods, &key);
343 if (key == TMPI_UNDEFINED)
345 color = TMPI_UNDEFINED;
348 tMPI_Comm_split(comm_old, color, key, comm_cart);
350 tMPI_Cart_init(comm_cart, ndims, dims, periods);
356 int tMPI_Cart_sub(tMPI_Comm comm, int *remain_dims, tMPI_Comm *newcomm)
362 int *oldcoords = NULL;
364 int ndims_notused = 1;
365 int color_notused = 0;
368 tMPI_Trace_print("tMPI_Cart_sub(%p, %p, %p)", comm, remain_dims, newcomm);
370 tMPI_Comm_rank(comm, &myrank);
373 oldcoords = (int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
374 dims = (int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
375 periods = (int*)tMPI_Malloc(sizeof(int)*comm->cart->ndims);
377 /* get old coordinates */
378 tMPI_Cart_coords(comm, myrank, comm->cart->ndims, oldcoords);
380 for (i = 0; i < comm->cart->ndims; i++)
384 /* for the remaining dimensions, copy dimensionality data */
385 dims[ndims] = comm->cart->dims[i];
386 periods[ndims] = comm->cart->periods[i];
391 /* base color on not used coordinates. We keep a
392 ndims_notused index multiplier.*/
393 color_notused += oldcoords[i]*ndims_notused;
394 ndims_notused *= comm->cart->dims[i];
399 /* key=myrank, because we want the order to remain the same */
400 tMPI_Comm_split(comm, color_notused, myrank, newcomm);
401 tMPI_Cart_init(newcomm, ndims, dims, periods);