2 This source code file is part of thread_mpi.
3 Written by Sander Pronk, Erik Lindahl, and possibly others.
5 Copyright (c) 2009, Sander Pronk, Erik Lindahl.
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10 1) Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
12 2) Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15 3) Neither the name of the copyright holders nor the
16 names of its contributors may be used to endorse or promote products
17 derived from this software without specific prior written permission.
19 THIS SOFTWARE IS PROVIDED BY US ''AS IS'' AND ANY
20 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL WE BE LIABLE FOR ANY
23 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 If you want to redistribute modifications, please consider that
31 scientific software is very special. Version control is crucial -
32 bugs must be traceable. We will be happy to consider code for
33 inclusion in the official distribution, but derived work should not
34 be called official thread_mpi. Details are found in the README & COPYING
38 #ifndef TMPI_ATOMIC_H_
39 #define TMPI_ATOMIC_H_
43 * \brief Atomic operations for fast SMP synchronization
45 * This file defines atomic integer operations and spinlocks for
46 * fast synchronization in performance-critical regions.
48 * In general, the best option is to use functions without explicit
49 * locking, e.g. tMPI_Atomic_fetch_add() or tMPI_Atomic_cas().
51 * Depending on the architecture/compiler, these operations may either
52 * be provided as functions or macros; be aware that those macros may
53 * reference their arguments repeatedly, possibly leading to multiply
54 * evaluated code with side effects: be careful with what you use as
57 * Not all architectures support atomic operations though inline assembly,
58 * and even if they do it might not be implemented here. In that case
59 * we use a fallback mutex implementation, so you can always count on
60 * the function interfaces working.
62 * Don't use spinlocks in non-performance-critical regions like file I/O.
63 * Since they always spin busy they would waste CPU cycles instead of
64 * properly yielding to a computation thread while waiting for the disk.
66 * Finally, note that all our spinlock operations are defined to return
67 * 0 if initialization or locking completes successfully.
68 * This is the opposite of some other implementations, but the same standard
69 * as used for pthread mutexes. So, if e.g. are trying to lock a spinlock,
70 * you will have gotten the lock if the return value is 0.
72 * tMPI_Spinlock_islocked(x) obviously still returns 1 if the lock is locked,
73 * and 0 if it is available, though...
75 /* Se the comments on the non-atomic versions for explanations */
79 #include "visibility.h"
86 } /* Avoids screwing up auto-indentation */
89 /* Setting TMPI_ATOMICS_DISABLED permits the build to enforce that no
90 * atomic operations are used. This is used when building to run
93 * It could also be useful as a temporary measure on some
94 * compiler+hardware for which the detection below fails to produce a
95 * correct result. Performance will be greatly improved by using
96 * whatever atomic operations are available, so make sure such a
97 * measure is only temporary! */
98 #ifdef TMPI_ATOMICS_DISABLED
101 #define TMPI_NO_ATOMICS
106 /* first check for gcc/icc platforms.
107 Some compatible compilers, like icc on linux+mac will take this path,
109 #if ( (defined(__GNUC__) || defined(__PATHSCALE__) || defined(__PGI)) && \
110 (!defined(__xlc__)) && (!defined(_CRAYC)) && (!defined(TMPI_TEST_NO_ATOMICS)) )
113 #define TMPI_GCC_VERSION (__GNUC__ * 10000 \
114 + __GNUC_MINOR__ * 100 \
115 + __GNUC_PATCHLEVEL__)
118 /* now check specifically for several architectures: */
119 #if ((defined(i386) || defined(__x86_64__)) && !defined(__OPEN64__))
121 #include "atomic/gcc_x86.h"
123 #elif (defined(__ia64__))
125 #include "atomic/gcc_ia64.h"
127 /* for now we use gcc intrinsics on gcc: */
128 /*#elif (defined(__powerpc__) || (defined(__ppc__)) )*/
129 /*#include "atomic/gcc_ppc.h"*/
131 #elif defined(__FUJITSU) && ( defined(__sparc_v9__) || defined (__sparcv9) )
133 /* Fujitsu FX10 SPARC compiler */
134 #include "atomic/fujitsu_sparc64.h"
137 /* otherwise, there's a generic gcc intrinsics version: */
138 #include "atomic/gcc.h"
140 #endif /* end of check for gcc specific architectures */
143 #elif (defined(_MSC_VER) && (_MSC_VER >= 1200) && \
144 (!defined(TMPI_TEST_NO_ATOMICS)) )
146 /* Microsoft Visual C on x86, define taken from FFTW who got it from
147 Morten Nissov. icc on windows will take this path. */
148 #include "atomic/msvc.h"
150 #elif ( (defined(__IBM_GCC_ASM) || defined(__IBM_STDCPP_ASM)) && \
151 (defined(__powerpc__) || defined(__ppc__)) && \
152 (!defined(TMPI_TEST_NO_ATOMICS)) )
154 /* PowerPC using xlC intrinsics. */
156 #include "atomic/xlc_ppc.h"
158 #elif ( ( defined(__xlC__) || defined(__xlc__) ) && \
159 (!defined(TMPI_TEST_NO_ATOMICS)) )
160 /* IBM xlC compiler */
161 #include "atomic/xlc_ppc.h"
164 #elif (defined (__sun) && (defined(__sparcv9) || defined(__sparc)) && \
165 (!defined(TMPI_TEST_NO_ATOMICS)) )
166 /* Solaris on SPARC (Sun C Compiler, Solaris Studio) */
167 #include "atomic/suncc-sparc.h"
169 #elif defined(__FUJITSU) && defined(__sparc__)
171 /* Fujitsu FX10 SPARC compiler requires gcc compatibility with -Xg */
172 #error Atomics support for Fujitsu FX10 compiler requires -Xg (gcc compatibility)
174 #elif defined(_CRAYC)
177 #include "atomic/cce.h"
181 /** Indicates that no support for atomic operations is present. */
182 #define TMPI_NO_ATOMICS
185 #endif /* platform-specific checks */
187 #endif /* TMPI_NO_ATOMICS */
189 #ifdef TMPI_NO_ATOMICS
191 /* No atomic operations, use mutex fallback. Documentation is in x86 section */
193 #ifdef TMPI_CHECK_ATOMICS
194 #error No atomic operations implemented for this cpu/compiler combination.
198 /** Memory barrier operation
200 Modern CPUs rely heavily on out-of-order execution, and one common feature
201 is that load/stores might be reordered. Also, when using inline assembly
202 the compiler might already have loaded the variable we are changing into
203 a register, so any update to memory won't be visible.
205 This command creates a memory barrier, i.e. all memory results before
206 it in the code should be visible to all memory operations after it - the
207 CPU cannot propagate load/stores across it.
209 This barrier is a full barrier: all load and store operations of
210 instructions before it are completed, while all load and store operations
211 that are in instructions after it won't be done before this barrier.
215 #define tMPI_Atomic_memory_barrier()
217 /** Memory barrier operation with acquire semantics
219 This barrier is a barrier with acquire semantics: the terminology comes
220 from its common use after acquiring a lock: all load/store instructions
221 after this barrier may not be re-ordered to happen before this barrier.
225 #define tMPI_Atomic_memory_barrier_acq()
227 /** Memory barrier operation with release semantics
229 This barrier is a barrier with release semantics: the terminology comes
230 from its common use before releasing a lock: all load/store instructions
231 before this barrier may not be re-ordered to happen after this barrier.
235 #define tMPI_Atomic_memory_barrier_rel()
238 /* signal that they exist */
239 #define TMPI_HAVE_ACQ_REL_BARRIERS
242 /** Atomic operations datatype
244 * Portable synchronization primitives like mutexes are effective for
245 * many purposes, but usually not very high performance.
246 * One of the problem is that you have the overhead of a function call,
247 * and another is that Mutexes often have extra overhead to make the
248 * scheduling fair. Finally, if performance is important we don't want
249 * to suspend the thread if we cannot lock a mutex, but spin-lock at 100%
250 * CPU usage until the resources is available (e.g. increment a counter).
252 * These things can often be implemented with inline-assembly or other
253 * system-dependent functions, and we provide such functionality for the
254 * most common platforms. For portability we also have a fallback
255 * implementation using a mutex for locking.
257 * Performance-wise, the fastest solution is always to avoid locking
258 * completely (obvious, but remember it!). If you cannot do that, the
259 * next best thing is to use atomic operations that e.g. increment a
260 * counter without explicit locking. Spinlocks are useful to lock an
261 * entire region, but leads to more overhead and can be difficult to
262 * debug - it is up to you to make sure that only the thread owning the
265 * You should normally NOT use atomic operations for things like
266 * I/O threads. These should yield to other threads while waiting for
267 * the disk instead of spinning at 100% CPU usage.
269 * It is imperative that you use the provided routines for reading
270 * and writing, since some implementations require memory barriers before
271 * the CPU or memory sees an updated result. The structure contents is
272 * only visible here so it can be inlined for performance - it might
273 * change without further notice.
275 * \note No initialization is required for atomic variables.
277 * Currently, we have (real) atomic operations for:
279 * - gcc version 4.1 and later (all platforms)
280 * - x86 or x86_64, using GNU compilers
281 * - x86 or x86_64, using Intel compilers
282 * - x86 or x86_64, using Pathscale compilers
283 * - Itanium, using GNU compilers
284 * - Itanium, using Intel compilers
285 * - Itanium, using HP compilers
286 * - PowerPC, using GNU compilers
287 * - PowerPC, using IBM AIX compilers
288 * - PowerPC, using IBM compilers >=7.0 under Linux or Mac OS X.
289 * - Sparc64, using Fujitsu compilers.
295 * - tMPI_Atomic_add_return
296 * - tMPI_Atomic_fetch_add
298 typedef struct tMPI_Atomic
300 int value; /**< The atomic value.*/
305 /** Atomic pointer type equivalent to tMPI_Atomic_t
307 * Useful for lock-free and wait-free data structures.
308 * The only operations available for this type are:
310 * - tMPI_Atomic_ptr_get
311 * - tMPI_Atomic_ptr_set
312 * - tMPI_Atomic_ptr_cas
314 typedef struct tMPI_Atomic_ptr
316 void *value; /**< The atomic pointer. */
323 * Spinlocks provide a faster synchronization than mutexes,
324 * although they consume CPU-cycles while waiting. They are implemented
325 * with atomic operations and inline assembly whenever possible, and
326 * otherwise we use a fallback implementation where a spinlock is identical
327 * to a mutex (this is one of the reasons why you have to initialize them).
329 * There are no guarantees whatsoever about fair scheduling or
330 * debugging if you make a mistake and unlock a variable somebody
331 * else has locked - performance is the primary goal of spinlocks.
334 * - tMPI_Spinlock_init
335 * - tMPI_Spinlock_lock
336 * - tMPI_Spinlock_unlock
337 * - tMPI_Spinlock_trylock
338 * - tMPI_Spinlock_wait
340 typedef struct tMPI_Spinlock *tMPI_Spinlock_t;
342 /*! \def TMPI_SPINLOCK_INITIALIZER
343 * \brief Spinlock static initializer
345 * This is used for static spinlock initialization, and has the same
346 * properties as TMPI_THREAD_MUTEX_INITIALIZER has for mutexes.
347 * This is only for inlining in the tMPI_Thread.h header file. Whether
348 * it is 0, 1, or something else when unlocked depends on the platform.
349 * Don't assume anything about it. It might even be a mutex when using the
350 * fallback implementation!
354 #define TMPI_SPINLOCK_INITIALIZER { NULL }
356 /* Since mutexes guarantee memory barriers this works fine */
357 /** Return value of an atomic integer
359 * Also implements proper memory barriers when necessary.
360 * The actual implementation is system-dependent.
362 * \param a Atomic variable to read
363 * \return Integer value of the atomic variable
368 int tMPI_Atomic_get(const tMPI_Atomic_t *a);
370 /** Write value to an atomic integer
372 * Also implements proper memory barriers when necessary.
373 * The actual implementation is system-dependent.
375 * \param a Atomic variable
376 * \param i Integer to set the atomic variable to.
381 void tMPI_Atomic_set(tMPI_Atomic_t *a, int i);
384 /** Return value of an atomic pointer
386 * Also implements proper memory barriers when necessary.
387 * The actual implementation is system-dependent.
389 * \param a Atomic variable to read
390 * \return Pointer value of the atomic variable
395 void* tMPI_Atomic_ptr_get(const tMPI_Atomic_ptr_t *a);
400 /** Write value to an atomic pointer
402 * Also implements proper memory barriers when necessary.
403 * The actual implementation is system-dependent.
405 * \param a Atomic variable
406 * \param p Pointer value to set the atomic variable to.
411 void tMPI_Atomic_ptr_set(tMPI_Atomic_ptr_t *a, void *p);
413 /** Add integer to atomic variable
415 * Also implements proper memory barriers when necessary.
416 * The actual implementation is system-dependent.
418 * \param a atomic datatype to modify
419 * \param i integer to increment with. Use i<0 to subtract atomically.
421 * \return The new value (after summation).
424 int tMPI_Atomic_add_return(tMPI_Atomic_t *a, int i);
426 #define TMPI_ATOMIC_HAVE_NATIVE_ADD_RETURN
431 /** Add to variable, return the old value.
433 * This operation is quite useful for synchronization counters.
434 * By performing a fetchadd with N, a thread can e.g. reserve a chunk
435 * with the next N iterations, and the return value is the index
436 * of the first element to treat.
438 * Also implements proper memory barriers when necessary.
439 * The actual implementation is system-dependent.
441 * \param a atomic datatype to modify
442 * \param i integer to increment with. Use i<0 to subtract atomically.
444 * \return The value of the atomic variable before addition.
447 int tMPI_Atomic_fetch_add(tMPI_Atomic_t *a, int i);
449 #define TMPI_ATOMIC_HAVE_NATIVE_FETCH_ADD
454 /** Atomic compare-and-swap operation
456 * The \a old value is compared with the memory value in the atomic datatype.
457 * If the are identical, the atomic type is swapped with the new value,
458 * and otherwise left unchanged.
460 * This is *the* synchronization primitive: it has a consensus number of
461 * infinity, and is available in some form on all modern CPU architectures.
462 * In the words of Herlihy&Shavit (The art of multiprocessor programming),
463 * it is the 'king of all wild things'.
465 * In practice, use it as follows: You can start by reading a value
466 * (without locking anything), perform some calculations, and then
467 * atomically try to update it in memory unless it has changed. If it has
468 * changed you will get an error return code - reread the new value
469 * an repeat the calculations in that case.
471 * \param a Atomic datatype ('memory' value)
472 * \param old_val Integer value read from the atomic type at an earlier point
473 * \param new_val New value to write to the atomic type if it currently is
474 * identical to the old value.
476 * \return True (1) if the swap occurred: i.e. if the value in a was equal
477 * to old_val. False (0) if the swap didn't occur and the value
478 * was not equal to old_val.
480 * \note The exchange occured if the return value is identical to \a old.
483 int tMPI_Atomic_cas(tMPI_Atomic_t *a, int old_val, int new_val);
488 /** Atomic pointer compare-and-swap operation
490 * The \a old value is compared with the memory value in the atomic datatype.
491 * If the are identical, the atomic type is swapped with the new value,
492 * and otherwise left unchanged.
494 * This is essential for implementing wait-free lists and other data
495 * structures. See 'tMPI_Atomic_cas()'.
497 * \param a Atomic datatype ('memory' value)
498 * \param old_val Pointer value read from the atomic type at an earlier point
499 * \param new_val New value to write to the atomic type if it currently is
500 * identical to the old value.
502 * \return True (1) if the swap occurred: i.e. if the value in a was equal
503 * to old_val. False (0) if the swap didn't occur and the value
504 * was not equal to old_val.
506 * \note The exchange occured if the return value is identical to \a old.
509 int tMPI_Atomic_ptr_cas(tMPI_Atomic_ptr_t * a, void *old_val,
512 /** Atomic swap operation.
514 Atomically swaps the data in the tMPI_Atomic_t operand with the value of b.
515 Note: This has no good assembly counterparts on many architectures, so
516 it might not be faster than a repreated CAS.
518 \param a Pointer to atomic type
519 \param b Value to swap
520 \return the original value of a
523 int tMPI_Atomic_swap(tMPI_Atomic_t *a, int b);
525 /** Atomic swap pointer operation.
527 Atomically swaps the pointer in the tMPI_Atomic_ptr_t operand with the
529 Note: This has no good assembly counterparts on many architectures, so
530 it might not be faster than a repreated CAS.
532 \param a Pointer to atomic type
533 \param b Value to swap
534 \return the original value of a
537 void *tMPI_Atomic_ptr_swap(tMPI_Atomic_ptr_t *a, void *b);
539 #define TMPI_ATOMIC_HAVE_NATIVE_SWAP
543 /** Initialize spinlock
545 * In theory you can call this from multiple threads, but remember
546 * that we don't check for errors. If the first thread proceeded to
547 * lock the spinlock after initialization, the second will happily
548 * overwrite the contents and unlock it without warning you.
550 * \param x Spinlock pointer.
555 void tMPI_Spinlock_init( tMPI_Spinlock_t *x);
557 #define TMPI_ATOMIC_HAVE_NATIVE_SPINLOCK
562 * This routine blocks until the spinlock is available, and
563 * the locks it again before returning.
565 * \param x Spinlock pointer
568 void tMPI_Spinlock_lock( tMPI_Spinlock_t *x);
571 /** Attempt to acquire spinlock
573 * This routine acquires the spinlock if possible, but if
574 * already locked it return an error code immediately.
576 * \param x Spinlock pointer
578 * \return 0 if the mutex was available so we could lock it,
579 * otherwise a non-zero integer (1) if the lock is busy.
582 int tMPI_Spinlock_trylock( tMPI_Spinlock_t *x);
586 * \param x Spinlock pointer
588 * Unlocks the spinlock, regardless if which thread locked it.
591 void tMPI_Spinlock_unlock( tMPI_Spinlock_t *x);
595 /** Check if spinlock is locked
597 * This routine returns immediately with the lock status.
599 * \param x Spinlock pointer
601 * \return 1 if the spinlock is locked, 0 otherwise.
604 int tMPI_Spinlock_islocked( tMPI_Spinlock_t *x);
606 /** Wait for a spinlock to become available
608 * This routine blocks until the spinlock is unlocked,
609 * but in contrast to tMPI_Spinlock_lock() it returns without
610 * trying to lock the spinlock.
612 * \param x Spinlock pointer
615 void tMPI_Spinlock_wait(tMPI_Spinlock_t *x);
618 #endif /* TMPI_NO_ATOMICS */
620 /* now define all the atomics that are not avaible natively. These
621 are done on the assumption that a native CAS does exist. */
622 #include "atomic/derived.h"
624 /* this allows us to use the inline keyword without breaking support for
625 some compilers that don't support it: */
626 #ifdef inline_defined_in_atomic
630 #if !defined(TMPI_NO_ATOMICS) && !defined(TMPI_ATOMICS)
631 /* Set it here to make sure the user code can check this without having to have
633 /** Indicates that support for atomic operations is present. */
643 #endif /* TMPI_ATOMIC_H_ */