2 This source code file is part of thread_mpi.
3 Written by Sander Pronk, Erik Lindahl, and possibly others.
5 Copyright (c) 2009, Sander Pronk, Erik Lindahl.
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10 1) Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
12 2) Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15 3) Neither the name of the copyright holders nor the
16 names of its contributors may be used to endorse or promote products
17 derived from this software without specific prior written permission.
19 THIS SOFTWARE IS PROVIDED BY US ''AS IS'' AND ANY
20 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL WE BE LIABLE FOR ANY
23 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 If you want to redistribute modifications, please consider that
31 scientific software is very special. Version control is crucial -
32 bugs must be traceable. We will be happy to consider code for
33 inclusion in the official distribution, but derived work should not
34 be called official thread_mpi. Details are found in the README & COPYING
43 * \brief Partial implementation of MPI using only threads.
45 * See the MPI specification at
46 * http://www.mpi-forum.org/docs/docs.html
47 * for an explanation of what these functions do.
49 * Because this is a thread-based library, be very careful with global
50 * variables and static variables in functions: they will be shared across
51 * all threads and lead to conflicts if not properly mutex-ed or barrier-ed
54 * \sa http://www.mpi-forum.org/docs/docs.html for MPI documentation.
58 /* for size_t, include stddef.h - which is in C89. This is done
59 regardless of whether we're compiling C++ or C code because the base
60 library for this is in C. */
63 #include "visibility.h"
70 } /* Avoids screwing up auto-indentation */
77 Use this to check for thread_mpi with the preprocessor. */
81 /** tMPI initialization thread affinity strategy.
83 Used in the tMPI_Init_affinity() and tMPI_Init_fn_affinity() functions,
84 to control how affinity is set. The default tMPI_Init() and tMPI_Init_fn()
85 functions use the TMPI_AFFINITY_ALL_CORES strategy.
87 These strategies are fairly basic. For more flexibility, use the
88 tMPI_Set_affinity() function.*/
91 TMPI_AFFINITY_NONE=0, /**< Do not set any thread affinity */
92 TMPI_AFFINITY_ALL_CORES, /**< Only set affinity if the number of threads
93 is equal to the number of hardware threads
94 (cores + hyperthreads). This is the only
95 safe way to set thread affinity,
96 without clashes between multiple
97 instances of the same program. */
98 } tMPI_Affinity_strategy;
102 /** tMPI Communicator
104 Holds the group of processes to communicate
105 with, and defines the scope for global operations such as broadcast. */
106 typedef struct tmpi_comm_ *tMPI_Comm;
110 The group structure. Contains a list of threads. */
111 typedef struct tmpi_group_ *tMPI_Group;
115 Request structure for holding data about non-blocking transfers. */
116 typedef struct tmpi_req_ *tMPI_Request;
121 tMPI data type structure. Holds info about datatypes. */
122 typedef struct tmpi_datatype_ *tMPI_Datatype;
125 /*! \name tMPI Data types
126 These are MPI data types as specified by the MPI standard.
127 Note that not all are available. */
130 extern const tMPI_Datatype TMPI_CHAR; /**< char */
132 extern const tMPI_Datatype TMPI_SHORT; /**< short */
134 extern const tMPI_Datatype TMPI_INT; /**< int */
136 extern const tMPI_Datatype TMPI_LONG; /**< long */
137 #ifdef SIZEOF_LONG_LONG_INT
139 extern const tMPI_Datatype TMPI_LONG_LONG; /**< long long */
141 extern const tMPI_Datatype TMPI_LONG_LONG_INT; /**< long long int */
144 extern const tMPI_Datatype TMPI_SIGNED_CHAR; /**< signed char */
146 extern const tMPI_Datatype TMPI_UNSIGNED_CHAR; /**< unsigned char */
148 extern const tMPI_Datatype TMPI_UNSIGNED_SHORT; /**< unsigned short */
150 extern const tMPI_Datatype TMPI_UNSIGNED; /**< unsigned int */
152 extern const tMPI_Datatype TMPI_UNSIGNED_LONG; /**< unsigned long */
153 #ifdef SIZEOF_LONG_LONG_INT
155 extern const tMPI_Datatype TMPI_UNSIGNED_LONG_LONG; /**< unsigned long long */
158 extern const tMPI_Datatype TMPI_FLOAT; /**< float */
160 extern const tMPI_Datatype TMPI_DOUBLE; /**< double */
162 extern const tMPI_Datatype TMPI_LONG_DOUBLE; /**< long double */
163 /*extern tMPI_Datatype tMPI_UNSIGNED_WCHAR */
165 extern const tMPI_Datatype TMPI_BYTE; /**< byte (for binary
168 extern const tMPI_Datatype TMPI_POINTER; /**< pointer (thread_mpi
177 TMPI_SUCCESS=0, /*!< No error */
178 TMPI_ERR_MALLOC, /*!< Out of memory */
179 TMPI_ERR_INIT, /*!< Initialization error */
180 TMPI_ERR_FINALIZE, /*!< Finalize error */
181 TMPI_ERR_GROUP, /*!< Group error */
182 TMPI_ERR_COMM, /*!< Comm error */
183 TMPI_ERR_STATUS, /*!< Status error */
184 TMPI_ERR_GROUP_RANK, /*!< Group rank error */
187 TMPI_ERR_CART_CREATE_NPROCS,
188 TMPI_ERR_XFER_COUNTERPART,
189 TMPI_ERR_XFER_BUFSIZE,
190 TMPI_ERR_XFER_BUF_OVERLAP,
194 TMPI_ERR_MULTI_MISMATCH,
199 TMPI_ERR_PROCNR, /*!< Hardware processor number (such as for
200 thread affinity) error */
203 N_TMPI_ERR /* this must be the last one */
206 /** Maximum length of error string for tMPI_Error_string() */
207 #define TMPI_MAX_ERROR_STRING 256
209 /** default code for undefined value,
211 For example for undefined color in tMPI_Split(). */
212 #define TMPI_UNDEFINED -1
214 /** error handler function */
215 typedef void (*tMPI_Errhandler_fn)(tMPI_Comm*, int*);
216 /** error handler object */
217 typedef struct tmpi_errhandler_ *tMPI_Errhandler;
219 /** pre-defined error handler that abort()s on every error */
220 extern tMPI_Errhandler TMPI_ERRORS_ARE_FATAL;
221 /** pre-defined error handler that tries to continue on every error */
222 extern tMPI_Errhandler TMPI_ERRORS_RETURN;
224 /*! \name tMPI_Comm_compare() return codes */
226 /** Identical comms*/
228 /** Comms with the same members in the same order*/
229 #define TMPI_CONGRUENT 1
230 /** Comms with the same members in the different order*/
231 #define TMPI_SIMILAR 2
232 /** Comms with the different members */
233 #define TMPI_UNEQUAL 3
237 /** Source number wildcard so tMPI_Recv(), etc. can receive from
239 #define TMPI_ANY_SOURCE -1
240 /** Tag number wildcard so tMPI_Recv(), etc. can receive messages with
242 #define TMPI_ANY_TAG -1
244 /** Return code for Cartesian topology with tMPI_Topo_test(). */
246 /** Return code for graph topology with tMPI_Topo_test(). */
250 /** Pre-initialized communicator with all available threads. */
252 extern tMPI_Comm TMPI_COMM_WORLD;
255 /** A pre-defined NULL communicator to compare against, to check comm
257 #define TMPI_COMM_NULL NULL
258 /** A pre-defined NULL group to compare against, to check group
260 #define TMPI_GROUP_NULL NULL
262 /** the empty group */
263 extern tMPI_Group TMPI_GROUP_EMPTY;
266 /** The maximum processor name returned using tMPI_Get_processor_name(). */
267 #define TMPI_MAX_PROCESSOR_NAME 128
270 /** Used as NULL status for tMPI_Recv(), etc. */
271 #define TMPI_STATUS_IGNORE NULL
272 /** Used as NULL status list for tMPI_Waitall(), etc. */
273 #define TMPI_STATUSES_IGNORE NULL
277 Holds status info (tag, sender, amount of data transmitted) for receives.
278 The status object is user-maintained. */
279 typedef struct tmpi_status_
281 int TMPI_SOURCE; /**< Message source rank. */
282 int TMPI_TAG; /**< Message source tag. */
283 int TMPI_ERROR; /**< Message error. */
284 size_t transferred; /**< Number of transferred bytes */
285 int cancelled; /**< Whether the transmission was canceled */
287 /*typedef struct tmpi_status_ tMPI_Status;*/
290 #define TMPI_REQUEST_NULL NULL
292 /** collective communication special to signify that the send
293 buffer is to function as receive buffer.
295 Used, for example in tMPI_Reduce. */
296 #define TMPI_IN_PLACE NULL
299 /** tMPI_Reduce operators.
301 These all work (except obviously bad combinations like bitwise
302 and/or/xor on floats, etc): */
305 TMPI_MAX, /**< calculate maximum value */
306 TMPI_MIN, /**< calculate minimum value */
307 TMPI_SUM, /**< calculate sum */
308 TMPI_PROD, /**< calculate product */
309 TMPI_LAND, /**< calculate logical and */
310 TMPI_BAND, /**< calculate binary and */
311 TMPI_LOR, /**< calculate logical or */
312 TMPI_BOR, /**< calculate binary or */
313 TMPI_LXOR, /**< calculate logical xor */
314 TMPI_BXOR /**< calculate binary xor */
318 /* function to obtain tMPI_COMM_SELF */
319 tMPI_Comm tMPI_Get_comm_self(void);
321 /** The thread-specific comm containing only the thread itself.
324 \return the self comm object associated with the thread. */
325 #define TMPI_COMM_SELF (tMPI_Get_comm_self())
335 /*! \name Initialization and exit functions
337 /** Traditional MPI initializer; spawns threads that start at the given
340 Seeks the argument '-nt n', where n is the number of
341 threads that will be created. If n==0, the number of threads will
342 be the recommended number of threads for this platform as obtained
343 from tMPI_Get_recommended_ntreads().
345 The new threads then run the function start_function, with the original
346 argc and argv. This function could be main(), or any other function;
347 calling this function again - whether from the started threads or from
348 the main thread - has no effect.
350 On platforms that support thread affinity setting, this function will
351 use the 'all-cores' affinity strategy: it will only set thread affinity
352 if the number of threads is equal to the number of hardware threads
353 (cores + hyperthreads).
355 \param[in] argc argc of original main() invocation, or NULL
356 \param[in] argv argv of original main() invocation, or NULL.
357 \param[in] start_function Starting function of type
358 int start_function(int argc, char *argv[]);
360 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
362 int tMPI_Init(int *argc, char ***argv,
363 int (*start_function)(int, char**));
366 /** Generic init function thread MPI intializer and thread spawner.
368 Creates N threads (including main thread)
369 that run the function start_function, which takes a void* argument,
370 given by arg. The function start_function also gets called by the main
371 thread. When the function start_function returns it, will behave
372 as if tMPI_Finalize is called, and if it's a sub-thread it will
375 If N==0, the number of threads will be the recommended number of
376 threads for this platform as obtained from tMPI_Get_recommended_ntreads().
378 Note that thread affinity strategy only has an effect when this is
379 supported by the underlying platform. As of yet (2012), this is not the
380 case for Mac OS X, for example.
382 \param[in] main_thread_returns whether the control in the main thread
383 should return immediately (if true), or
384 the start_function() should be called
385 from the main thread, too (if false).
386 \param[in] N The number of threads to start (or 0 to
387 automatically determine this).
388 \param[in] aff_strategy The thread affinity strategy to use.
389 \param[in] start_function The function to start threads at
390 (including main thread if
391 main_thread_returns).
392 \param[in] arg An optional argument for start_function().
394 \return TMPI_FAILURE on failure, TMPI_SUCCESS on succes (after all
395 threads have finished if main_thread_returns=true). */
397 int tMPI_Init_fn(int main_thread_returns, int N,
398 tMPI_Affinity_strategy aff_strategy,
399 void (*start_function)(void*), void *arg);
405 /** get the number of threads from the command line
407 can be called before tMPI_Init()
409 \param[in] argc argc from main()
410 \param[in] argv argv from main()
411 \param[in] optname name of the argument specifying the
412 number of threads to run. If this is
413 NULL, this function will read the first
414 argument and interpret it as the number
416 \param[out] nthreads the number of threads
418 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
420 int tMPI_Get_N(int *argc, char ***argv, const char *optname, int *nthreads);
424 /** Waits for all other threads to finish and cleans up
426 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
428 int tMPI_Finalize(void);
431 /** Just kills all threads.
433 Not really neccesary because exit() would do that for us anyway.
435 \param[in] comm Comm to kill threads for
436 \param[in] errorcode Error code to exit with
438 \return Never returns. */
440 int tMPI_Abort(tMPI_Comm comm, int errorcode);
442 /** whether tMPI_Init, but not yet tMPI_Finalize, has been run
444 \param[out] flag Set to TRUE if tMPI_Init() has been called,
447 \return always returns TMPI_SUCCESS. */
449 int tMPI_Initialized(int *flag);
451 /** Determine whether tMPI_Finalize has been run.
453 \param[out] flag Set to TRUE if tMPI_Finalize() has been
454 called, FALSE if not.
456 \return always returns TMPI_SUCCESS. */
458 int tMPI_Finalized(int *flag);
469 /*! \name Error handling functions
471 /** Create an error handler object from a function.
473 \param[in] function The function to make an error handler of.
474 \param[out] errhandler The error handler.
476 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
478 int tMPI_Create_errhandler(tMPI_Errhandler_fn *function,
479 tMPI_Errhandler *errhandler);
482 /** Free the error handler object.
484 \param[in] errhandler The error handler.
485 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
487 int tMPI_Errhandler_free(tMPI_Errhandler *errhandler);
489 /** Set the error handler.
491 \param[in] comm the communicator to set the error handler for.
492 \param[in] errhandler the error handler.
494 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
496 int tMPI_Comm_set_errhandler(tMPI_Comm comm, tMPI_Errhandler errhandler);
498 /** get the error handler.
500 Gets the error handler associated with a comm
502 \param[in] comm the communicator to get the error handler for.
503 \param[out] errhandler the error handler.
505 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
507 int tMPI_Comm_get_errhandler(tMPI_Comm comm, tMPI_Errhandler *errhandler);
509 /** get the error string associated with an error code.
511 The length of the error string will never exceed TMPI_MAX_ERROR_STRING.
513 \param[in] errorcode The error code.
514 \param[out] string The pre-allocated char pointer to output to.
515 \param[out] resultlen The length of the error string. Will
516 never be longer than TMPI_MAX_ERROR_STRING.
518 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
520 int tMPI_Error_string(int errorcode, char *string, size_t *resultlen);
530 /*! \name Environment query functions
532 /** returns string with thread number.
534 \param[out] name Pre-allocated string to output name to (will not
535 be longer than TMPI_MAX_PROCESSOR_NAME).
536 \param[out] resultlen The length of the output. Note that this is an
537 int instead of a size_t because the MPI standard
538 for some reason defines all sizes as int
540 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
542 int tMPI_Get_processor_name(char *name, int *resultlen);
544 /** get a time value as a double, in seconds.
549 double tMPI_Wtime(void);
550 /** get the resolution of tMPI_Wtime as a double, in seconds
552 \return time resolution. */
554 double tMPI_Wtick(void);
557 #define tMPI_This_threadnr() (int)(tMPI_Get_current() - threads)
559 /** Get the thread number of this thread.
560 Mostly for debugging.
562 \return the global thread number. */
563 int tMPI_This_Threadnr(void);
576 /*! \name tMPI_Group functions
578 /** Get the size (number of members) of a group.
580 \param[in] group The group.
581 \param[out] size Size.
582 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
584 int tMPI_Group_size(tMPI_Group group, int *size);
586 /** Get the rank of a thread in a group
588 \param[in] group The group.
589 \param[out] rank Variable for the rank, or TMPI_UNDEFINED
591 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
593 int tMPI_Group_rank(tMPI_Group group, int *rank);
595 /** Create a new group as a the collection of threads with given ranks.
597 \param[in] group The group from which the ranks are taken.
598 \param[in] n The number of new group members.
599 \param[in] ranks The ranks of the threads to add to the new group.
600 \param[out] newgroup The new group.
602 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
604 int tMPI_Group_incl(tMPI_Group group, int n, int *ranks, tMPI_Group *newgroup);
606 /** Get a pointer to the group in the comm.
608 \param[in] comm The comm from which to take the group.
609 \param[out] group The comm's group.
611 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
613 int tMPI_Comm_group(tMPI_Comm comm, tMPI_Group *group);
615 /** De-allocate a group
617 \param[in] group The group to de-allocate.
618 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
620 int tMPI_Group_free(tMPI_Group *group);
629 /*! \name tMPI_Comm functions
631 /** Get the comm size (nr. of threads).
633 \param[in] comm The comm to query.
634 \param[out] size The comm size.
635 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
637 int tMPI_Comm_size(tMPI_Comm comm, int *size);
639 /** get the rank in comm of the current process
641 \param[in] comm The comm to query.
642 \param[out] rank Thread rank in comm.
643 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
645 int tMPI_Comm_rank(tMPI_Comm comm, int *rank);
647 /** Compare two comms. Returns TMPI_IDENT if the two comms point to
648 the same underlying comm structure, TMPI_CONGRUENT if all
649 members appear in the both comms in the same order, TMPI_SIMILAR
650 if both comms have the smae members but not in the same order, or
651 TMPI_UNEQUAL if the comms have different members.
653 \param[in] comm1 The first comm to compare.
654 \param[in] comm2 The second comm to compare.
655 \param[out] result The output result, one of the values
657 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
659 int tMPI_Comm_compare(tMPI_Comm comm1, tMPI_Comm comm2, int *result);
662 /** De-allocate a comm
666 \param[in] comm The comm to free.
667 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
669 int tMPI_Comm_free(tMPI_Comm *comm);
671 /** Create a comm based on group membership.
673 Collective function that creates a new comm containing only proceses
674 that are members of the given group.
676 \param[in] comm The originating comm.
677 \param[in] group The group of threads to create a comm from.
678 \param[out] newcomm The new comm.
680 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
682 int tMPI_Comm_create(tMPI_Comm comm, tMPI_Group group, tMPI_Comm *newcomm);
684 /** Split up a group into same-colored sub-groups ordered by key.
686 This is the main comm creation function: it's a collective call that takes
687 a color and a key from each process, and arranges all threads that
688 call tMPI_Split() withe the same color together into a comm.
690 The rank in the new group will be based on the value given in key.
692 Passing TMPI_UNDEFINED as a color will result in the thread not being
693 part of any group, and getting TMPI_COMM_NULL back in newcomm.
695 \param[in] comm The originating comm.
696 \param[in] color This thread's color (determines which comm it will
697 be in). Giving TMPI_UNDEFINED will result in
698 this thread not being in any group.
699 \param[in] key This thread's key (determines rank).
700 \param[out] newcomm The new comm.
701 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
703 int tMPI_Comm_split(tMPI_Comm comm, int color, int key, tMPI_Comm *newcomm);
705 /** Make a duplicate of a comm.
709 \param[in] comm The originating comm.
710 \param[in] newcomm The new comm.
711 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
713 int tMPI_Comm_dup(tMPI_Comm comm, tMPI_Comm *newcomm);
723 /*! \name Topology functions
725 /* topology functions */
726 /** Check what type of topology the comm has.
728 \param[in] comm The comm to query
729 \param[out] status The type of topology.
731 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
733 int tMPI_Topo_test(tMPI_Comm comm, int *status);
735 /** Get the dimensionality of a comm with a topology.
737 \param[in] comm The comm to query.
738 \param[out] ndims The number of dimensions.
740 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
743 int tMPI_Cartdim_get(tMPI_Comm comm, int *ndims);
744 /** Get the size and pbc a of a comm with a Cartesian topology has.
746 \param[in] comm The comm to query.
747 \param[in] maxdims The maximum number of dimensions in the periods
748 and coords parameter.
749 \param[out] dims The number of dimensions.
750 \param[out] periods The periodicity in each dimension.
751 \param[out] coords The number of coordinates in each dimension.
753 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
756 int tMPI_Cart_get(tMPI_Comm comm, int maxdims, int *dims, int *periods,
760 /** Get rank that a specific set of process coordinates has in
761 a Cartesian topology.
763 \param[in] comm The comm to query.
764 \param[in] coords The coordinates in each dimension.
765 \param[out] rank The rank associated with the coordinates.
767 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
769 int tMPI_Cart_rank(tMPI_Comm comm, int *coords, int *rank);
771 /** Get coordinates of a process rank in a Cartesian topology.
773 \param[in] comm The comm to query.
774 \param[in] rank The rank associated with the coordinates.
775 \param[in] maxdims The maximum number of dimensions in the coords
777 \param[out] coords The coordinates in each dimension.
779 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
781 int tMPI_Cart_coords(tMPI_Comm comm, int rank, int maxdims, int *coords);
783 /** Get optimal rank this process would have in a Cartesian topology.
785 \param[in] comm The comm to query.
786 \param[in] ndims The number of dimensions.
787 \param[in] dims The size in each dimension.
788 \param[in] periods The periodicity in each dimension.
790 \param[out] newrank The rank the thread would have given the topology.
792 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
794 int tMPI_Cart_map(tMPI_Comm comm, int ndims, int *dims, int *periods,
797 /** Create a comm with a Cartesian topology.
799 \param[in] comm_old The originating comm.
800 \param[in] ndims The number of dimensions.
801 \param[in] dims The size in each dimension.
802 \param[in] periods The periodicity in each dimension.
803 \param[in] reorder Whether to allow reordering of the threads
804 according to tMPI_Cart_map().
805 \param[out] comm_cart The new comm with Cartesian topology.
807 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
809 int tMPI_Cart_create(tMPI_Comm comm_old, int ndims, int *dims, int *periods,
810 int reorder, tMPI_Comm *comm_cart);
812 /** Create a comms that are sub-spaces of the Cartesian topology communicator.
813 Works like a MPI_Comm_split() for the Cartesian dimensions specified
814 as false in remain_dims.
816 \param[in] comm The originating comm with Cartesian topology.
817 \param[in] remain_dims An Boolean array that decides whether a specific
818 dimensionality should remain in newcomm (if true),
819 or should be split up (if false).
820 \param[out] newcomm The new split communicator
822 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
824 int tMPI_Cart_sub(tMPI_Comm comm, int *remain_dims, tMPI_Comm *newcomm);
835 /*! \name Data type manipulation functions
837 /** Create a contiguous data type (the only type possible right now).
839 Creates a datatype that is a vector of oldtype.
841 \param[in] count The number of oldtype types in the new type.
842 \param[in] oldtype The old data type.
843 \param[out] newtype The new data type (still needs to be committed).
844 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
846 int tMPI_Type_contiguous(int count, tMPI_Datatype oldtype,
847 tMPI_Datatype *newtype);
850 /** Make a data type ready for use.
852 \param[in,out] datatype The new datatype.
853 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
855 int tMPI_Type_commit(tMPI_Datatype *datatype);
865 /*! \name Point-to-point communication functions
868 /* blocking transfers. The actual transfer (copy) is done on the receiving end
869 (so that the receiver's cache already contains the data that it presumably
871 /** Send message; blocks until buf is reusable.
873 \param[in] buf The buffer with data to send.
874 \param[in] count The number of items to send.
875 \param[in] datatype The data type of the items in buf.
876 \param[in] dest The rank of the destination thread.
877 \param[in] tag The message tag.
878 \param[in] comm The shared communicator.
879 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
881 int tMPI_Send(void* buf, int count, tMPI_Datatype datatype, int dest,
882 int tag, tMPI_Comm comm);
884 /** Receive message; blocks until buf is filled.
886 \param[out] buf The buffer for data to receive.
887 \param[in] count The maximum number of items to receive.
888 \param[in] datatype The data type of the items in buf.
889 \param[in] source The rank of the source thread (or TMPI_ANY_SOURCE).
890 \param[in] tag The message tag (or TMPI_ANY_TAG).
891 \param[in] comm The shared communicator.
892 \param[out] status The message status.
893 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
895 int tMPI_Recv(void* buf, int count, tMPI_Datatype datatype, int source,
896 int tag, tMPI_Comm comm, tMPI_Status *status);
898 /** Send & receive message at the same time.
900 Blocks until recvbuf is filled, and sendbuf is ready for reuse.
902 \param[in] sendbuf The buffer with data to send.
903 \param[in] sendcount The number of items to send.
904 \param[in] sendtype The data type of the items in send buf.
905 \param[in] dest The rank of the destination thread.
906 \param[in] sendtag The send message tag.
907 \param[out] recvbuf The buffer for data to receive.
908 \param[in] recvcount The maximum number of items to receive.
909 \param[in] recvtype The data type of the items in recvbuf.
910 \param[in] source The rank of the source thread (or TMPI_ANY_SOURCE).
911 \param[in] recvtag The recveive message tag (or TMPI_ANY_TAG).
912 \param[in] comm The shared communicator.
913 \param[out] status The received message status.
914 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
916 int tMPI_Sendrecv(void *sendbuf, int sendcount, tMPI_Datatype sendtype,
917 int dest, int sendtag, void *recvbuf, int recvcount,
918 tMPI_Datatype recvtype, int source, int recvtag,
919 tMPI_Comm comm, tMPI_Status *status);
921 /* async send/recv. The actual transfer is done on the receiving
922 end, during tMPI_Wait, tMPI_Waitall or tMPI_Test. For tMPI_Waitall,
923 the incoming messages are processed in the order they come in. */
925 /** Initiate sending a message, non-blocking.
927 This makes the buffer available to be received. The contents of buf
928 should not be touched before the transmission is finished with
929 tMPI_Wait(), tMPI_Test() or tMPI_Waitall().
932 \param[in] buf The buffer with data to send.
933 \param[in] count The number of items to send.
934 \param[in] datatype The data type of the items in buf.
935 \param[in] dest The rank of the destination thread.
936 \param[in] tag The message tag.
937 \param[in] comm The shared communicator.
938 \param[out] request The request object that can be used in tMPI_Wait(),
940 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
942 int tMPI_Isend(void* buf, int count, tMPI_Datatype datatype, int dest,
943 int tag, tMPI_Comm comm, tMPI_Request *request);
945 /** Initiate receiving a message.
947 This makes the buffer available to be filled with data. The contents of
948 buf should not be relied on before the transmission is finished with
949 tMPI_Wait(), tMPI_Test() or tMPI_Waitall().
951 \param[out] buf The buffer for data to receive.
952 \param[in] count The maximum number of items to receive.
953 \param[in] datatype The data type of the items in buf.
954 \param[in] source The rank of the source thread (or TMPI_ANY_SOURCE).
955 \param[in] tag The message tag (or TMPI_ANY_TAG).
956 \param[in] comm The shared communicator.
957 \param[out] request The request object that can be used in tMPI_Wait(),
959 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
961 int tMPI_Irecv(void* buf, int count, tMPI_Datatype datatype, int source,
962 int tag, tMPI_Comm comm, tMPI_Request *request);
967 /** Test whether a message is transferred.
969 \param[in,out] request The request obtained wit tMPI_Isend()/tMPI_Irecv().
970 \param[out] flag A flag set to TRUE(1) if the request is finished,
972 \param[out] status Message status (can be set to TMPI_STATUS_IGNORE).
974 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
976 int tMPI_Test(tMPI_Request *request, int *flag, tMPI_Status *status);
978 /** Wait until a message is transferred.
980 \param[in,out] request The request obtained wit tMPI_Isend()/tMPI_Irecv().
981 \param[out] status Message status.
983 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
985 int tMPI_Wait(tMPI_Request *request, tMPI_Status *status);
990 /** Wait until several messages are transferred.
992 \param[in] count The number of requests
993 \param[in,out] array_of_requests List of count requests obtained with
994 tMPI_Isend()/tMPI_Irecv().
995 \param[out] array_of_statuses List of count message statuses (can
996 be set to TMPI_STATUSES_IGNORE).
998 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1000 int tMPI_Waitall(int count, tMPI_Request *array_of_requests,
1001 tMPI_Status *array_of_statuses);
1003 /** Test whether several messages are transferred.
1005 \param[in] count The number of requests
1006 \param[in,out] array_of_requests List of count requests obtained with
1007 tMPI_Isend()/tMPI_Irecv().
1008 \param[out] flag Whether all requests have completed.
1009 \param[out] array_of_statuses List of count message statuses (can
1010 be set to TMPI_STATUSES_IGNORE).
1012 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1014 int tMPI_Testall(int count, tMPI_Request *array_of_requests, int *flag,
1015 tMPI_Status *array_of_statuses);
1017 /** Wait until one of several messages is transferred.
1019 \param[in] count The number of requests
1020 \param[in,out] array_of_requests List of count requests obtained with
1021 tMPI_Isend()/tMPI_Irecv().
1022 \param[out] index Index of the request that has
1024 \param[out] status Pointer to tMPI_Status object
1025 associated with completed request.
1027 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1029 int tMPI_Waitany(int count, tMPI_Request *array_of_requests,
1030 int *index, tMPI_Status *status);
1032 /** Test whether one of several messages is transferred.
1034 \param[in] count The number of requests
1035 \param[in,out] array_of_requests List of count requests obtained with
1036 tMPI_Isend()/tMPI_Irecv().
1037 \param[out] index Index of the request that has
1039 \param[out] flag Whether any request has completed.
1040 \param[out] status Pointer to tMPI_Status object
1041 associated with completed request.
1043 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1045 int tMPI_Testany(int count, tMPI_Request *array_of_requests,
1046 int *index, int *flag, tMPI_Status *status);
1048 /** Wait until some of several messages are transferred. Waits until at least
1049 one message is transferred.
1051 \param[in] incount The number of requests
1052 \param[in,out] array_of_requests List of count requests obtained with
1053 tMPI_Isend()/tMPI_Irecv().
1054 \param[out] outcount Number of completed requests
1055 \param[out] array_of_indices Array of ints that gets filled with
1056 the indices of the completed requests.
1057 \param[out] array_of_statuses List of count message statuses (can
1058 be set to TMPI_STATUSES_IGNORE).
1060 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1062 int tMPI_Waitsome(int incount, tMPI_Request *array_of_requests,
1063 int *outcount, int *array_of_indices,
1064 tMPI_Status *array_of_statuses);
1066 /** Test whether some of several messages are transferred.
1068 \param[in] incount The number of requests
1069 \param[in,out] array_of_requests List of count requests obtained with
1070 tMPI_Isend()/tMPI_Irecv().
1071 \param[out] outcount Number of completed requests
1072 \param[out] array_of_indices Array of ints that gets filled with
1073 the indices of the completed requests.
1074 \param[out] array_of_statuses List of count message statuses (can
1075 be set to TMPI_STATUSES_IGNORE).
1077 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1079 int tMPI_Testsome(int incount, tMPI_Request *array_of_requests,
1080 int *outcount, int *array_of_indices,
1081 tMPI_Status *array_of_statuses);
1088 /** get the number of actually transferred items from a receive
1091 \param[in] status The status.
1092 \param[in] datatype The data type which was received.
1093 \param[out] count The number of items actually received.
1095 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1097 int tMPI_Get_count(tMPI_Status *status, tMPI_Datatype datatype, int *count);
1107 /*! \name Synchronization functions
1109 /** Block until all threads in the comm call this function.
1111 \param[in] comm The comm object.
1113 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1115 int tMPI_Barrier(tMPI_Comm comm);
1124 /*! \name Multicast communication functions
1126 /** Broadcast from one thread to all others in comm.
1128 Collective function; data is transferred from root's buffer to all others'
1131 \param[in,out] buffer The buffer to send from (root)/receive from
1133 \param[in] count The number of items to send/receive.
1134 \param[in] datatype The type of the items to send/receive.
1135 \param[in] root The rank of the sending thread.
1136 \param[in] comm The communicator.
1138 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1140 int tMPI_Bcast(void* buffer, int count, tMPI_Datatype datatype, int root,
1143 /** Gather data from all threads in comm to root.
1145 Collective function; assumes that all data is received in blocks of
1148 \param[in] sendbuf The send buffer for all threads (root may
1149 specify TMPI_IN_PLACE, in which case it
1150 transfers nothing to itself).
1151 \param[in] sendcount The number of items to send for all threads.
1152 \param[in] sendtype The type of the items to send.
1153 \param[out] recvbuf The receiving buffer (for root thread).
1154 \param[in] recvcount The number of items to receive (for root).
1155 \param[in] recvtype The type of the items to receive (for root).
1156 \param[in] root The rank of root.
1157 \param[in] comm The communicator.
1159 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1161 int tMPI_Gather(void* sendbuf, int sendcount, tMPI_Datatype sendtype,
1162 void* recvbuf, int recvcount, tMPI_Datatype recvtype, int root,
1166 /** Gather irregularly laid out data from all processes in comm to root.
1168 Collective function.
1170 \param[in] sendbuf The send buffer for all threads (root may
1171 specify TMPI_IN_PLACE, in which case it
1172 transfers nothing to itself).
1173 \param[in] sendcount The number of items to send for all threads.
1174 \param[in] sendtype The type of the items to send.
1175 \param[out] recvbuf The receiving buffer (for root thread).
1176 \param[in] recvcounts The list of number of items to receive (for
1178 \param[in] displs The list of displacements in recvbuf to
1179 receive data in (for root).
1180 \param[in] recvtype The type of the items to receive (for root).
1181 \param[in] root The rank of root.
1182 \param[in] comm The communicator.
1184 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1186 int tMPI_Gatherv(void* sendbuf, int sendcount, tMPI_Datatype sendtype,
1187 void* recvbuf, int *recvcounts, int *displs,
1188 tMPI_Datatype recvtype, int root, tMPI_Comm comm);
1191 /** Spread parts of sendbuf to all processes in comm from root.
1193 Collective function.
1195 \param[in] sendbuf The send buffer for root.
1196 \param[in] sendcount The number of items for root to send to each
1198 \param[in] sendtype The type of the items root sends.
1199 \param[out] recvbuf The receiving buffer for all receiving threads
1200 (root may specify TMPI_IN_PLACE, in which case
1201 it transmits nothing to itself).
1202 \param[in] recvcount The number of items recvbuf can receive.
1203 \param[in] recvtype The type of items to receive.
1204 \param[in] root The rank of root.
1205 \param[in] comm The communicator.
1207 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1209 int tMPI_Scatter(void* sendbuf, int sendcount, tMPI_Datatype sendtype,
1210 void* recvbuf, int recvcount, tMPI_Datatype recvtype, int root,
1213 /** Spread irregularly laid out parts of sendbuf to all processes
1216 Collective function.
1218 \param[in] sendbuf The send buffer for root.
1219 \param[in] sendcounts List of the number of items for root to send
1221 \param[in] displs List of displacements in sendbuf from which
1222 to start transmission to each thread.
1223 \param[in] sendtype The type of the items root sends.
1224 \param[out] recvbuf The receiving buffer for all receiving threads
1225 (root may specify TMPI_IN_PLACE, in which case
1226 it transmits nothing to itself).
1227 \param[in] recvcount The number of items recvbuf can receive.
1228 \param[in] recvtype The type of items to receive.
1229 \param[in] root The rank of root.
1230 \param[in] comm The communicator.
1232 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1234 int tMPI_Scatterv(void* sendbuf, int *sendcounts, int *displs,
1235 tMPI_Datatype sendtype, void* recvbuf, int recvcount,
1236 tMPI_Datatype recvtype, int root, tMPI_Comm comm);
1239 /** Spread out parts of sendbuf to all processes from all processes in
1242 Collective function.
1244 \param[in] sendbuf The send buffer.
1245 \param[in] sendcount The number of items for to send to each thread.
1246 \param[in] sendtype The type of the items to send.
1247 \param[out] recvbuf The receive buffer for all threads.
1248 \param[in] recvcount The number of items recvbuf can receive per
1250 \param[in] recvtype The type of items to receive.
1251 \param[in] comm The communicator.
1253 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1255 int tMPI_Alltoall(void* sendbuf, int sendcount, tMPI_Datatype sendtype,
1256 void* recvbuf, int recvcount, tMPI_Datatype recvtype,
1260 /** Spread out irregularly laid out parts of sendbuf to all
1261 processes from all processes in comm.
1263 Collective function.
1265 \param[in] sendbuf The send buffer.
1266 \param[in] sendcounts List of the number of items for to send to
1268 \param[in] sdispls List of the displacements in sendbuf of items
1269 to send to each thread.
1270 \param[in] sendtype The type of the items to send.
1271 \param[out] recvbuf The receive buffer for all threads.
1272 \param[in] recvcounts List of the number of items recvbuf can
1273 receive from each thread.
1274 \param[in] rdispls List of the displacements in recvbuf of items
1275 to receive from each thread.
1276 \param[in] recvtype The type of items to receive.
1277 \param[in] comm The communicator.
1279 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1281 int tMPI_Alltoallv(void* sendbuf, int *sendcounts, int *sdispls,
1282 tMPI_Datatype sendtype, void* recvbuf, int *recvcounts,
1283 int *rdispls, tMPI_Datatype recvtype, tMPI_Comm comm);
1295 /*! \name Reduce functions
1297 /** Do an operation between all locally held buffers on all items in the
1298 buffers, and send the results to root.
1300 Collective function.
1302 \param[in] sendbuf The operand parameters. Root may specify
1303 TMPI_IN_PLACE, in which case recvbuf will hold
1304 the operand parameters.
1305 \param[out] recvbuf The result buffer at root.
1306 \param[in] count The number of items to do operation on.
1307 \param[in] datatype The data type of the items.
1308 \param[in] op The operation to perform.
1309 \param[in] root The root thread (which is to receive the results).
1310 \param[in] comm The communicator.
1312 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1314 int tMPI_Reduce(void* sendbuf, void* recvbuf, int count,
1315 tMPI_Datatype datatype, tMPI_Op op, int root, tMPI_Comm comm);
1319 /** Do an operation between all locally held buffers on all items in the
1320 buffers and broadcast the results.
1322 Collective function.
1325 \param[in] sendbuf The operand parameters. Any process may specify
1326 TMPI_IN_PLACE, in which case recvbuf will hold
1327 the operand parameters for that process.
1328 \param[in,out] recvbuf The result buffer.
1329 \param[in] count The number of items to do operation on.
1330 \param[in] datatype The data type of the items.
1331 \param[in] op The operation to perform.
1332 \param[in] comm The communicator.
1334 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1336 int tMPI_Allreduce(void* sendbuf, void* recvbuf, int count,
1337 tMPI_Datatype datatype, tMPI_Op op, tMPI_Comm comm);
1339 /** Do an tMPI_Reduce, but with the following assumption:
1340 recvbuf points to a valid buffer in all calling threads, or
1341 sendbuf has the value TMPI_IN_PLACE (in which case the values of
1342 sendbuf may be changed in that thread).
1344 This avoids unnecesary memory allocations associated with the normal
1347 Collective function.
1349 \param[in] sendbuf The operand parameters (or TMPI_IN_PLACE,
1350 in which case the operand parameters will
1352 \param[in,out] recvbuf The result buffer.
1353 \param[in] count The number of items to do operation on.
1354 \param[in] datatype The data type of the items.
1355 \param[in] op The operation to perform.
1356 \param[in] root The root thread (which is to receive
1358 \param[in] comm The communicator.
1360 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1362 int tMPI_Reduce_fast(void* sendbuf, void* recvbuf, int count,
1363 tMPI_Datatype datatype, tMPI_Op op, int root,
1366 /** Do a partial reduce operation, based on rank: the results of the
1367 reduction operation of ranks 0 - i will be put in the recvbuf of
1370 Collective function.
1372 \param[in] sendbuf The operand parameters. All ranks may specify
1373 TMPI_IN_PLACE, in which case recvbuf will hold
1374 the operand parameters.
1375 \param[in,out] recvbuf The result buffer.
1376 \param[in] count The number of items to do operation on.
1377 \param[in] datatype The data type of the items.
1378 \param[in] op The operation to perform.
1379 \param[in] comm The communicator.
1381 \return TMPI_SUCCESS on success, TMPI_FAILURE on failure. */
1383 int tMPI_Scan(void* sendbuf, void* recvbuf, int count,
1384 tMPI_Datatype datatype, tMPI_Op op, tMPI_Comm comm);
1392 } /* closing extern "C" */
1395 #endif /* TMPI_TMPI_H_ */