2 This source code file is part of thread_mpi.
3 Written by Sander Pronk, Erik Lindahl, and possibly others.
5 Copyright (c) 2009, Sander Pronk, Erik Lindahl.
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10 1) Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
12 2) Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15 3) Neither the name of the copyright holders nor the
16 names of its contributors may be used to endorse or promote products
17 derived from this software without specific prior written permission.
19 THIS SOFTWARE IS PROVIDED BY US ''AS IS'' AND ANY
20 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL WE BE LIABLE FOR ANY
23 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 If you want to redistribute modifications, please consider that
31 scientific software is very special. Version control is crucial -
32 bugs must be traceable. We will be happy to consider code for
33 inclusion in the official distribution, but derived work should not
34 be called official thread_mpi. Details are found in the README & COPYING
38 #ifndef TMPI_ATOMIC_H_
39 #define TMPI_ATOMIC_H_
43 * \brief Atomic operations for fast SMP synchronization
45 * This file defines atomic integer operations and spinlocks for
46 * fast synchronization in performance-critical regions.
48 * In general, the best option is to use functions without explicit
49 * locking, e.g. tMPI_Atomic_fetch_add() or tMPI_Atomic_cas().
51 * Depending on the architecture/compiler, these operations may either
52 * be provided as functions or macros; be aware that those macros may
53 * reference their arguments repeatedly, possibly leading to multiply
54 * evaluated code with side effects: be careful with what you use as
57 * Not all architectures support atomic operations though inline assembly,
58 * and even if they do it might not be implemented here. In that case
59 * we use a fallback mutex implementation, so you can always count on
60 * the function interfaces working.
62 * Don't use spinlocks in non-performance-critical regions like file I/O.
63 * Since they always spin busy they would waste CPU cycles instead of
64 * properly yielding to a computation thread while waiting for the disk.
66 * Finally, note that all our spinlock operations are defined to return
67 * 0 if initialization or locking completes successfully.
68 * This is the opposite of some other implementations, but the same standard
69 * as used for pthread mutexes. So, if e.g. are trying to lock a spinlock,
70 * you will have gotten the lock if the return value is 0.
72 * tMPI_Spinlock_islocked(x) obviously still returns 1 if the lock is locked,
73 * and 0 if it is available, though...
75 /* Se the comments on the non-atomic versions for explanations */
79 #include "visibility.h"
86 } /* Avoids screwing up auto-indentation */
89 /* Setting TMPI_ATOMICS_DISABLED permits the build to enforce that no
90 * atomic operations are used. This is used when building to run
93 * It could also be useful as a temporary measure on some
94 * compiler+hardware for which the detection below fails to produce a
95 * correct result. Performance will be greatly improved by using
96 * whatever atomic operations are available, so make sure such a
97 * measure is only temporary! */
98 #ifdef TMPI_ATOMICS_DISABLED
101 #define TMPI_NO_ATOMICS
106 /* first check for gcc/icc platforms.
107 Some compatible compilers, like icc on linux+mac will take this path,
109 #if ( (defined(__GNUC__) || defined(__PATHSCALE__) || defined(__PGI)) && \
110 (!defined(__xlc__)) && (!defined(_CRAYC)) && (!defined(TMPI_TEST_NO_ATOMICS)) )
113 #define TMPI_GCC_VERSION (__GNUC__ * 10000 \
114 + __GNUC_MINOR__ * 100 \
115 + __GNUC_PATCHLEVEL__)
118 /* now check specifically for several architectures: */
119 #if ((defined(__i386__) || defined(__x86_64__)) && !defined(__OPEN64__))
121 #include "atomic/gcc_x86.h"
123 #elif (defined(__ia64__))
125 #include "atomic/gcc_ia64.h"
127 /* for now we use gcc intrinsics on gcc: */
128 /*#elif (defined(__powerpc__) || (defined(__ppc__)) )*/
129 /*#include "atomic/gcc_ppc.h"*/
131 #elif defined(__FUJITSU) && ( defined(__sparc_v9__) || defined (__sparcv9) )
133 /* Fujitsu FX10 SPARC compiler */
134 #include "atomic/fujitsu_sparc64.h"
137 /* otherwise, there's a generic gcc intrinsics version: */
138 #include "atomic/gcc.h"
140 #endif /* end of check for gcc specific architectures */
143 #elif (defined(_MSC_VER) && (_MSC_VER >= 1200) && \
144 (!defined(TMPI_TEST_NO_ATOMICS)) )
146 /* Microsoft Visual C on x86, define taken from FFTW who got it from
147 Morten Nissov. icc on windows will take this path. */
148 #include "atomic/msvc.h"
150 #elif ( (defined(__IBM_GCC_ASM) || defined(__IBM_STDCPP_ASM)) && \
151 (defined(__powerpc__) || defined(__ppc__)) && \
152 (!defined(TMPI_TEST_NO_ATOMICS)) )
154 /* PowerPC using xlC intrinsics. */
156 #include "atomic/xlc_ppc.h"
158 #elif ( ( defined(__xlC__) || defined(__xlc__) ) && \
159 (!defined(TMPI_TEST_NO_ATOMICS)) )
160 /* IBM xlC compiler */
161 #include "atomic/xlc_ppc.h"
164 #elif (defined (__sun) && (defined(__sparcv9) || defined(__sparc)) && \
165 (!defined(TMPI_TEST_NO_ATOMICS)) )
166 /* Solaris on SPARC (Sun C Compiler, Solaris Studio) */
167 #include "atomic/suncc-sparc.h"
169 #elif defined(__FUJITSU) && defined(__sparc__)
171 /* Fujitsu FX10 SPARC compiler requires gcc compatibility with -Xg */
172 #warning Atomics support for Fujitsu FX10 compiler requires -Xg (gcc compatibility)
173 #define TMPI_NO_ATOMICS
175 #elif defined(_CRAYC)
178 #include "atomic/cce.h"
182 /** Indicates that no support for atomic operations is present. */
183 #define TMPI_NO_ATOMICS
186 #endif /* platform-specific checks */
188 #endif /* TMPI_NO_ATOMICS */
190 #ifdef TMPI_NO_ATOMICS
192 /* No atomic operations, use mutex fallback. Documentation is in x86 section */
194 #ifdef TMPI_CHECK_ATOMICS
195 #error No atomic operations implemented for this cpu/compiler combination.
199 /** Memory barrier operation
201 Modern CPUs rely heavily on out-of-order execution, and one common feature
202 is that load/stores might be reordered. Also, when using inline assembly
203 the compiler might already have loaded the variable we are changing into
204 a register, so any update to memory won't be visible.
206 This command creates a memory barrier, i.e. all memory results before
207 it in the code should be visible to all memory operations after it - the
208 CPU cannot propagate load/stores across it.
210 This barrier is a full barrier: all load and store operations of
211 instructions before it are completed, while all load and store operations
212 that are in instructions after it won't be done before this barrier.
216 #define tMPI_Atomic_memory_barrier()
218 /** Memory barrier operation with acquire semantics
220 This barrier is a barrier with acquire semantics: the terminology comes
221 from its common use after acquiring a lock: all load/store instructions
222 after this barrier may not be re-ordered to happen before this barrier.
226 #define tMPI_Atomic_memory_barrier_acq()
228 /** Memory barrier operation with release semantics
230 This barrier is a barrier with release semantics: the terminology comes
231 from its common use before releasing a lock: all load/store instructions
232 before this barrier may not be re-ordered to happen after this barrier.
236 #define tMPI_Atomic_memory_barrier_rel()
239 /* signal that they exist */
240 #define TMPI_HAVE_ACQ_REL_BARRIERS
243 /** Atomic operations datatype
245 * Portable synchronization primitives like mutexes are effective for
246 * many purposes, but usually not very high performance.
247 * One of the problem is that you have the overhead of a function call,
248 * and another is that Mutexes often have extra overhead to make the
249 * scheduling fair. Finally, if performance is important we don't want
250 * to suspend the thread if we cannot lock a mutex, but spin-lock at 100%
251 * CPU usage until the resources is available (e.g. increment a counter).
253 * These things can often be implemented with inline-assembly or other
254 * system-dependent functions, and we provide such functionality for the
255 * most common platforms. For portability we also have a fallback
256 * implementation using a mutex for locking.
258 * Performance-wise, the fastest solution is always to avoid locking
259 * completely (obvious, but remember it!). If you cannot do that, the
260 * next best thing is to use atomic operations that e.g. increment a
261 * counter without explicit locking. Spinlocks are useful to lock an
262 * entire region, but leads to more overhead and can be difficult to
263 * debug - it is up to you to make sure that only the thread owning the
266 * You should normally NOT use atomic operations for things like
267 * I/O threads. These should yield to other threads while waiting for
268 * the disk instead of spinning at 100% CPU usage.
270 * It is imperative that you use the provided routines for reading
271 * and writing, since some implementations require memory barriers before
272 * the CPU or memory sees an updated result. The structure contents is
273 * only visible here so it can be inlined for performance - it might
274 * change without further notice.
276 * \note No initialization is required for atomic variables.
278 * Currently, we have (real) atomic operations for:
280 * - gcc version 4.1 and later (all platforms)
281 * - x86 or x86_64, using GNU compilers
282 * - x86 or x86_64, using Intel compilers
283 * - x86 or x86_64, using Pathscale compilers
284 * - Itanium, using GNU compilers
285 * - Itanium, using Intel compilers
286 * - Itanium, using HP compilers
287 * - PowerPC, using GNU compilers
288 * - PowerPC, using IBM AIX compilers
289 * - PowerPC, using IBM compilers >=7.0 under Linux or Mac OS X.
290 * - Sparc64, using Fujitsu compilers.
296 * - tMPI_Atomic_add_return
297 * - tMPI_Atomic_fetch_add
299 typedef struct tMPI_Atomic
301 int value; /**< The atomic value.*/
306 /** Atomic pointer type equivalent to tMPI_Atomic_t
308 * Useful for lock-free and wait-free data structures.
309 * The only operations available for this type are:
311 * - tMPI_Atomic_ptr_get
312 * - tMPI_Atomic_ptr_set
313 * - tMPI_Atomic_ptr_cas
315 typedef struct tMPI_Atomic_ptr
317 void *value; /**< The atomic pointer. */
324 * Spinlocks provide a faster synchronization than mutexes,
325 * although they consume CPU-cycles while waiting. They are implemented
326 * with atomic operations and inline assembly whenever possible, and
327 * otherwise we use a fallback implementation where a spinlock is identical
328 * to a mutex (this is one of the reasons why you have to initialize them).
330 * There are no guarantees whatsoever about fair scheduling or
331 * debugging if you make a mistake and unlock a variable somebody
332 * else has locked - performance is the primary goal of spinlocks.
335 * - tMPI_Spinlock_init
336 * - tMPI_Spinlock_lock
337 * - tMPI_Spinlock_unlock
338 * - tMPI_Spinlock_trylock
339 * - tMPI_Spinlock_wait
341 typedef struct tMPI_Spinlock *tMPI_Spinlock_t;
343 /*! \def TMPI_SPINLOCK_INITIALIZER
344 * \brief Spinlock static initializer
346 * This is used for static spinlock initialization, and has the same
347 * properties as TMPI_THREAD_MUTEX_INITIALIZER has for mutexes.
348 * This is only for inlining in the tMPI_Thread.h header file. Whether
349 * it is 0, 1, or something else when unlocked depends on the platform.
350 * Don't assume anything about it. It might even be a mutex when using the
351 * fallback implementation!
355 #define TMPI_SPINLOCK_INITIALIZER { NULL }
357 /* Since mutexes guarantee memory barriers this works fine */
358 /** Return value of an atomic integer
360 * Also implements proper memory barriers when necessary.
361 * The actual implementation is system-dependent.
363 * \param a Atomic variable to read
364 * \return Integer value of the atomic variable
369 int tMPI_Atomic_get(const tMPI_Atomic_t *a);
371 /** Write value to an atomic integer
373 * Also implements proper memory barriers when necessary.
374 * The actual implementation is system-dependent.
376 * \param a Atomic variable
377 * \param i Integer to set the atomic variable to.
382 void tMPI_Atomic_set(tMPI_Atomic_t *a, int i);
385 /** Return value of an atomic pointer
387 * Also implements proper memory barriers when necessary.
388 * The actual implementation is system-dependent.
390 * \param a Atomic variable to read
391 * \return Pointer value of the atomic variable
396 void* tMPI_Atomic_ptr_get(const tMPI_Atomic_ptr_t *a);
401 /** Write value to an atomic pointer
403 * Also implements proper memory barriers when necessary.
404 * The actual implementation is system-dependent.
406 * \param a Atomic variable
407 * \param p Pointer value to set the atomic variable to.
412 void tMPI_Atomic_ptr_set(tMPI_Atomic_ptr_t *a, void *p);
414 /** Add integer to atomic variable
416 * Also implements proper memory barriers when necessary.
417 * The actual implementation is system-dependent.
419 * \param a atomic datatype to modify
420 * \param i integer to increment with. Use i<0 to subtract atomically.
422 * \return The new value (after summation).
425 int tMPI_Atomic_add_return(tMPI_Atomic_t *a, int i);
427 #define TMPI_ATOMIC_HAVE_NATIVE_ADD_RETURN
432 /** Add to variable, return the old value.
434 * This operation is quite useful for synchronization counters.
435 * By performing a fetchadd with N, a thread can e.g. reserve a chunk
436 * with the next N iterations, and the return value is the index
437 * of the first element to treat.
439 * Also implements proper memory barriers when necessary.
440 * The actual implementation is system-dependent.
442 * \param a atomic datatype to modify
443 * \param i integer to increment with. Use i<0 to subtract atomically.
445 * \return The value of the atomic variable before addition.
448 int tMPI_Atomic_fetch_add(tMPI_Atomic_t *a, int i);
450 #define TMPI_ATOMIC_HAVE_NATIVE_FETCH_ADD
455 /** Atomic compare-and-swap operation
457 * The \a old value is compared with the memory value in the atomic datatype.
458 * If the are identical, the atomic type is swapped with the new value,
459 * and otherwise left unchanged.
461 * This is *the* synchronization primitive: it has a consensus number of
462 * infinity, and is available in some form on all modern CPU architectures.
463 * In the words of Herlihy&Shavit (The art of multiprocessor programming),
464 * it is the 'king of all wild things'.
466 * In practice, use it as follows: You can start by reading a value
467 * (without locking anything), perform some calculations, and then
468 * atomically try to update it in memory unless it has changed. If it has
469 * changed you will get an error return code - reread the new value
470 * an repeat the calculations in that case.
472 * \param a Atomic datatype ('memory' value)
473 * \param old_val Integer value read from the atomic type at an earlier point
474 * \param new_val New value to write to the atomic type if it currently is
475 * identical to the old value.
477 * \return True (1) if the swap occurred: i.e. if the value in a was equal
478 * to old_val. False (0) if the swap didn't occur and the value
479 * was not equal to old_val.
481 * \note The exchange occured if the return value is identical to \a old.
484 int tMPI_Atomic_cas(tMPI_Atomic_t *a, int old_val, int new_val);
489 /** Atomic pointer compare-and-swap operation
491 * The \a old value is compared with the memory value in the atomic datatype.
492 * If the are identical, the atomic type is swapped with the new value,
493 * and otherwise left unchanged.
495 * This is essential for implementing wait-free lists and other data
496 * structures. See 'tMPI_Atomic_cas()'.
498 * \param a Atomic datatype ('memory' value)
499 * \param old_val Pointer value read from the atomic type at an earlier point
500 * \param new_val New value to write to the atomic type if it currently is
501 * identical to the old value.
503 * \return True (1) if the swap occurred: i.e. if the value in a was equal
504 * to old_val. False (0) if the swap didn't occur and the value
505 * was not equal to old_val.
507 * \note The exchange occured if the return value is identical to \a old.
510 int tMPI_Atomic_ptr_cas(tMPI_Atomic_ptr_t * a, void *old_val,
513 /** Atomic swap operation.
515 Atomically swaps the data in the tMPI_Atomic_t operand with the value of b.
516 Note: This has no good assembly counterparts on many architectures, so
517 it might not be faster than a repreated CAS.
519 \param a Pointer to atomic type
520 \param b Value to swap
521 \return the original value of a
524 int tMPI_Atomic_swap(tMPI_Atomic_t *a, int b);
526 /** Atomic swap pointer operation.
528 Atomically swaps the pointer in the tMPI_Atomic_ptr_t operand with the
530 Note: This has no good assembly counterparts on many architectures, so
531 it might not be faster than a repreated CAS.
533 \param a Pointer to atomic type
534 \param b Value to swap
535 \return the original value of a
538 void *tMPI_Atomic_ptr_swap(tMPI_Atomic_ptr_t *a, void *b);
540 #define TMPI_ATOMIC_HAVE_NATIVE_SWAP
544 /** Initialize spinlock
546 * In theory you can call this from multiple threads, but remember
547 * that we don't check for errors. If the first thread proceeded to
548 * lock the spinlock after initialization, the second will happily
549 * overwrite the contents and unlock it without warning you.
551 * \param x Spinlock pointer.
556 void tMPI_Spinlock_init( tMPI_Spinlock_t *x);
558 #define TMPI_ATOMIC_HAVE_NATIVE_SPINLOCK
563 * This routine blocks until the spinlock is available, and
564 * the locks it again before returning.
566 * \param x Spinlock pointer
569 void tMPI_Spinlock_lock( tMPI_Spinlock_t *x);
572 /** Attempt to acquire spinlock
574 * This routine acquires the spinlock if possible, but if
575 * already locked it return an error code immediately.
577 * \param x Spinlock pointer
579 * \return 0 if the mutex was available so we could lock it,
580 * otherwise a non-zero integer (1) if the lock is busy.
583 int tMPI_Spinlock_trylock( tMPI_Spinlock_t *x);
587 * \param x Spinlock pointer
589 * Unlocks the spinlock, regardless if which thread locked it.
592 void tMPI_Spinlock_unlock( tMPI_Spinlock_t *x);
596 /** Check if spinlock is locked
598 * This routine returns immediately with the lock status.
600 * \param x Spinlock pointer
602 * \return 1 if the spinlock is locked, 0 otherwise.
605 int tMPI_Spinlock_islocked( tMPI_Spinlock_t *x);
607 /** Wait for a spinlock to become available
609 * This routine blocks until the spinlock is unlocked,
610 * but in contrast to tMPI_Spinlock_lock() it returns without
611 * trying to lock the spinlock.
613 * \param x Spinlock pointer
616 void tMPI_Spinlock_wait(tMPI_Spinlock_t *x);
619 #endif /* TMPI_NO_ATOMICS */
621 /* now define all the atomics that are not avaible natively. These
622 are done on the assumption that a native CAS does exist. */
623 #include "atomic/derived.h"
625 /* this allows us to use the inline keyword without breaking support for
626 some compilers that don't support it: */
627 #ifdef inline_defined_in_atomic
631 #if !defined(TMPI_NO_ATOMICS) && !defined(TMPI_ATOMICS)
632 /* Set it here to make sure the user code can check this without having to have
634 /** Indicates that support for atomic operations is present. */
644 #endif /* TMPI_ATOMIC_H_ */