2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 /* The macros in this file are intended to be used for writing
37 * architecture-independent SIMD intrinsics code.
38 * To support a new architecture, adding macros here should be (nearly)
42 #ifdef _gmx_simd_macros_h_
43 #error "gmx_simd_macros.h included twice"
45 #define _gmx_simd_macros_h_
47 /* NOTE: SSE2 acceleration does not include floor or blendv */
50 /* Uncomment the next line, without other SIMD active, for testing plain-C */
51 /* #define GMX_SIMD_REFERENCE_PLAIN_C */
52 #ifdef GMX_SIMD_REFERENCE_PLAIN_C
53 /* Plain C SIMD reference implementation, also serves as documentation */
54 #define GMX_HAVE_SIMD_MACROS
56 /* In general the reference SIMD supports any SIMD width, including 1.
57 * See types/nb_verlet.h for details
59 #define GMX_SIMD_REF_WIDTH 4
61 /* Include plain-C reference implementation, also serves as documentation */
62 #include "gmx_simd_ref.h"
64 #define GMX_SIMD_WIDTH_HERE GMX_SIMD_REF_WIDTH
66 /* float/double SIMD register type */
67 #define gmx_mm_pr gmx_simd_ref_pr
69 /* boolean SIMD register type */
70 #define gmx_mm_pb gmx_simd_ref_pb
72 /* integer SIMD register type, only for table indexing and exclusion masks */
73 #define gmx_epi32 gmx_simd_ref_epi32
74 #define GMX_SIMD_EPI32_WIDTH GMX_SIMD_REF_EPI32_WIDTH
76 /* Load GMX_SIMD_WIDTH_HERE reals for memory starting at r */
77 #define gmx_load_pr gmx_simd_ref_load_pr
78 /* Set all SIMD register elements to *r */
79 #define gmx_load1_pr gmx_simd_ref_load1_pr
80 #define gmx_set1_pr gmx_simd_ref_set1_pr
81 #define gmx_setzero_pr gmx_simd_ref_setzero_pr
82 #define gmx_store_pr gmx_simd_ref_store_pr
84 #define gmx_add_pr gmx_simd_ref_add_pr
85 #define gmx_sub_pr gmx_simd_ref_sub_pr
86 #define gmx_mul_pr gmx_simd_ref_mul_pr
87 /* For the FMA macros below, aim for c=d in code, so FMA3 uses 1 instruction */
88 #define gmx_madd_pr gmx_simd_ref_madd_pr
89 #define gmx_nmsub_pr gmx_simd_ref_nmsub_pr
91 #define gmx_max_pr gmx_simd_ref_max_pr
92 #define gmx_blendzero_pr gmx_simd_ref_blendzero_pr
94 #define gmx_round_pr gmx_simd_ref_round_pr
96 /* Not required, only used to speed up the nbnxn tabulated PME kernels */
97 #define GMX_SIMD_HAVE_FLOOR
98 #ifdef GMX_SIMD_HAVE_FLOOR
99 #define gmx_floor_pr gmx_simd_ref_floor_pr
102 /* Not required, only used when blendv is faster than comparison */
103 #define GMX_SIMD_HAVE_BLENDV
104 #ifdef GMX_SIMD_HAVE_BLENDV
105 #define gmx_blendv_pr gmx_simd_ref_blendv_pr
108 /* Copy the sign of a to b, assumes b >= 0 for efficiency */
109 #define gmx_cpsgn_nonneg_pr gmx_simd_ref_cpsgn_nonneg_pr
111 /* Very specific operation required in the non-bonded kernels */
112 #define gmx_masknot_add_pr gmx_simd_ref_masknot_add_pr
115 #define gmx_cmplt_pr gmx_simd_ref_cmplt_pr
117 /* Logical operations on SIMD booleans */
118 #define gmx_and_pb gmx_simd_ref_and_pb
119 #define gmx_or_pb gmx_simd_ref_or_pb
121 /* Returns a single int (0/1) which tells if any of the 4 booleans is True */
122 #define gmx_anytrue_pb gmx_simd_ref_anytrue_pb
124 /* Conversions only used for PME table lookup */
125 #define gmx_cvttpr_epi32 gmx_simd_ref_cvttpr_epi32
126 #define gmx_cvtepi32_pr gmx_simd_ref_cvtepi32_pr
128 /* These two function only need to be approximate, Newton-Raphson iteration
129 * is used for full accuracy in gmx_invsqrt_pr and gmx_inv_pr.
131 #define gmx_rsqrt_pr gmx_simd_ref_rsqrt_pr
132 #define gmx_rcp_pr gmx_simd_ref_rcp_pr
134 /* sqrt+inv+sin+cos+acos+atan2 are used for bonded potentials, exp for PME */
135 #define GMX_SIMD_HAVE_EXP
136 #ifdef GMX_SIMD_HAVE_EXP
137 #define gmx_exp_pr gmx_simd_ref_exp_pr
139 #define GMX_SIMD_HAVE_TRIGONOMETRIC
140 #ifdef GMX_SIMD_HAVE_TRIGONOMETRIC
141 #define gmx_sqrt_pr gmx_simd_ref_sqrt_pr
142 #define gmx_sincos_pr gmx_simd_ref_sincos_pr
143 #define gmx_acos_pr gmx_simd_ref_acos_pr
144 #define gmx_atan2_pr gmx_simd_ref_atan2_pr
147 #endif /* GMX_SIMD_REFERENCE_PLAIN_C */
150 /* The same SIMD macros can be translated to SIMD intrinsics (and compiled
151 * to instructions for) different SIMD width and float precision.
153 * On x86: The gmx_ prefix is replaced by _mm_ or _mm256_ (SSE or AVX).
154 * The _pr suffix is replaced by _ps or _pd (for single or double precision).
155 * Compiler settings will decide if 128-bit intrinsics will
156 * be translated into SSE or AVX instructions.
160 #ifdef GMX_USE_HALF_WIDTH_SIMD_HERE
161 #if defined GMX_X86_AVX_256
162 /* We have half SIMD width support, continue */
164 #error "half SIMD width intrinsics are not supported"
168 #ifdef GMX_TARGET_X86
171 /* This is for general x86 SIMD instruction sets that also support SSE2 */
172 #define GMX_HAVE_SIMD_MACROS
174 /* Include the highest supported x86 SIMD intrisics + math functions */
175 #ifdef GMX_X86_AVX_256
176 #include "gmx_x86_avx_256.h"
178 #include "gmx_math_x86_avx_256_double.h"
179 #else /* GMX_DOUBLE */
180 #include "gmx_math_x86_avx_256_single.h"
181 #endif /* GMX_DOUBLE */
182 #else /* GMX_X86_AVX_256 */
183 #ifdef GMX_X86_AVX_128_FMA
184 #include "gmx_x86_avx_128_fma.h"
186 #include "gmx_math_x86_avx_128_fma_double.h"
187 #else /* GMX_DOUBLE */
188 #include "gmx_math_x86_avx_128_fma_single.h"
189 #endif /* GMX_DOUBLE */
190 #else /* GMX_X86_AVX_128_FMA */
191 #ifdef GMX_X86_SSE4_1
192 #include "gmx_x86_sse4_1.h"
194 #include "gmx_math_x86_sse4_1_double.h"
195 #else /* GMX_DOUBLE */
196 #include "gmx_math_x86_sse4_1_single.h"
197 #endif /* GMX_DOUBLE */
198 #else /* GMX_X86_SSE4_1 */
200 #include "gmx_x86_sse2.h"
202 #include "gmx_math_x86_sse2_double.h"
203 #else /* GMX_DOUBLE */
204 #include "gmx_math_x86_sse2_single.h"
205 #endif /* GMX_DOUBLE */
206 #else /* GMX_X86_SSE2 */
207 #error No x86 acceleration defined
208 #endif /* GMX_X86_SSE2 */
209 #endif /* GMX_X86_SSE4_1 */
210 #endif /* GMX_X86_AVX_128_FMA */
211 #endif /* GMX_X86_AVX_256 */
213 /* exp and trigonometric functions are included above */
214 #define GMX_SIMD_HAVE_EXP
215 #define GMX_SIMD_HAVE_TRIGONOMETRIC
217 #if !defined GMX_X86_AVX_256 || defined GMX_USE_HALF_WIDTH_SIMD_HERE
221 #define GMX_SIMD_WIDTH_HERE 4
223 #define gmx_mm_pr __m128
225 #define gmx_mm_pb __m128
227 #define gmx_epi32 __m128i
228 #define GMX_SIMD_EPI32_WIDTH 4
230 #define gmx_load_pr _mm_load_ps
231 #define gmx_load1_pr _mm_load1_ps
232 #define gmx_set1_pr _mm_set1_ps
233 #define gmx_setzero_pr _mm_setzero_ps
234 #define gmx_store_pr _mm_store_ps
236 #define gmx_add_pr _mm_add_ps
237 #define gmx_sub_pr _mm_sub_ps
238 #define gmx_mul_pr _mm_mul_ps
239 #ifdef GMX_X86_AVX_128_FMA
240 #define GMX_SIMD_HAVE_FMA
241 #define gmx_madd_pr(a, b, c) _mm_macc_ps(a, b, c)
242 #define gmx_nmsub_pr(a, b, c) _mm_nmacc_ps(a, b, c)
244 #define gmx_madd_pr(a, b, c) _mm_add_ps(c, _mm_mul_ps(a, b))
245 #define gmx_nmsub_pr(a, b, c) _mm_sub_ps(c, _mm_mul_ps(a, b))
247 #define gmx_max_pr _mm_max_ps
248 #define gmx_blendzero_pr _mm_and_ps
250 #define gmx_cmplt_pr _mm_cmplt_ps
251 #define gmx_and_pb _mm_and_ps
252 #define gmx_or_pb _mm_or_ps
254 #ifdef GMX_X86_SSE4_1
255 #define gmx_round_pr(x) _mm_round_ps(x, 0x0)
256 #define GMX_SIMD_HAVE_FLOOR
257 #define gmx_floor_pr _mm_floor_ps
259 #define gmx_round_pr(x) _mm_cvtepi32_ps(_mm_cvtps_epi32(x))
262 #ifdef GMX_X86_SSE4_1
263 #define GMX_SIMD_HAVE_BLENDV
264 #define gmx_blendv_pr _mm_blendv_ps
267 static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
269 /* The value -0.0 has only the sign-bit set */
270 gmx_mm_pr sign_mask = _mm_set1_ps(-0.0);
271 return _mm_or_ps(_mm_and_ps(a, sign_mask), b);
274 static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
276 return _mm_add_ps(b, _mm_andnot_ps(a, c));
279 #define gmx_anytrue_pb _mm_movemask_ps
281 #define gmx_cvttpr_epi32 _mm_cvttps_epi32
282 #define gmx_cvtepi32_pr _mm_cvtepi32_ps
284 #define gmx_rsqrt_pr _mm_rsqrt_ps
285 #define gmx_rcp_pr _mm_rcp_ps
287 #define gmx_exp_pr gmx_mm_exp_ps
288 #define gmx_sqrt_pr gmx_mm_sqrt_ps
289 #define gmx_sincos_pr gmx_mm_sincos_ps
290 #define gmx_acos_pr gmx_mm_acos_ps
291 #define gmx_atan2_pr gmx_mm_atan2_ps
293 #else /* ifndef GMX_DOUBLE */
295 #define GMX_SIMD_WIDTH_HERE 2
297 #define gmx_mm_pr __m128d
299 #define gmx_mm_pb __m128d
301 #define gmx_epi32 __m128i
302 #define GMX_SIMD_EPI32_WIDTH 4
304 #define gmx_load_pr _mm_load_pd
305 #define gmx_load1_pr _mm_load1_pd
306 #define gmx_set1_pr _mm_set1_pd
307 #define gmx_setzero_pr _mm_setzero_pd
308 #define gmx_store_pr _mm_store_pd
310 #define gmx_add_pr _mm_add_pd
311 #define gmx_sub_pr _mm_sub_pd
312 #define gmx_mul_pr _mm_mul_pd
313 #ifdef GMX_X86_AVX_128_FMA
314 #define GMX_SIMD_HAVE_FMA
315 #define gmx_madd_pr(a, b, c) _mm_macc_pd(a, b, c)
316 #define gmx_nmsub_pr(a, b, c) _mm_nmacc_pd(a, b, c)
318 #define gmx_madd_pr(a, b, c) _mm_add_pd(c, _mm_mul_pd(a, b))
319 #define gmx_nmsub_pr(a, b, c) _mm_sub_pd(c, _mm_mul_pd(a, b))
321 #define gmx_max_pr _mm_max_pd
322 #define gmx_blendzero_pr _mm_and_pd
324 #ifdef GMX_X86_SSE4_1
325 #define gmx_round_pr(x) _mm_round_pd(x, 0x0)
326 #define GMX_SIMD_HAVE_FLOOR
327 #define gmx_floor_pr _mm_floor_pd
329 #define gmx_round_pr(x) _mm_cvtepi32_pd(_mm_cvtpd_epi32(x))
330 /* gmx_floor_pr is not used in code for pre-SSE4_1 hardware */
333 #ifdef GMX_X86_SSE4_1
334 #define GMX_SIMD_HAVE_BLENDV
335 #define gmx_blendv_pr _mm_blendv_pd
338 static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
340 gmx_mm_pr sign_mask = _mm_set1_pd(-0.0);
341 return _mm_or_pd(_mm_and_pd(a, sign_mask), b);
344 static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
346 return _mm_add_pd(b, _mm_andnot_pd(a, c));
349 #define gmx_cmplt_pr _mm_cmplt_pd
351 #define gmx_and_pb _mm_and_pd
352 #define gmx_or_pb _mm_or_pd
354 #define gmx_anytrue_pb _mm_movemask_pd
356 #define gmx_cvttpr_epi32 _mm_cvttpd_epi32
357 #define gmx_cvtepi32_pr _mm_cvtepi32_pd
359 #define gmx_rsqrt_pr(r) _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(r)))
360 #define gmx_rcp_pr(r) _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(r)))
362 #define gmx_exp_pr gmx_mm_exp_pd
363 #define gmx_sqrt_pr gmx_mm_sqrt_pd
364 #define gmx_sincos_pr gmx_mm_sincos_pd
365 #define gmx_acos_pr gmx_mm_acos_pd
366 #define gmx_atan2_pr gmx_mm_atan2_pd
368 #endif /* ifndef GMX_DOUBLE */
371 /* We have GMX_X86_AVX_256 and not GMX_USE_HALF_WIDTH_SIMD_HERE,
372 * so we use 256-bit SIMD.
377 #define GMX_SIMD_WIDTH_HERE 8
379 #define gmx_mm_pr __m256
381 #define gmx_mm_pb __m256
383 #define gmx_epi32 __m256i
384 #define GMX_SIMD_EPI32_WIDTH 8
386 #define gmx_load_pr _mm256_load_ps
387 #define gmx_load1_pr(x) _mm256_set1_ps((x)[0])
388 #define gmx_set1_pr _mm256_set1_ps
389 #define gmx_setzero_pr _mm256_setzero_ps
390 #define gmx_store_pr _mm256_store_ps
392 #define gmx_add_pr _mm256_add_ps
393 #define gmx_sub_pr _mm256_sub_ps
394 #define gmx_mul_pr _mm256_mul_ps
395 #define gmx_madd_pr(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
396 #define gmx_nmsub_pr(a, b, c) _mm256_sub_ps(c, _mm256_mul_ps(a, b))
397 #define gmx_max_pr _mm256_max_ps
398 #define gmx_blendzero_pr _mm256_and_ps
400 #define gmx_round_pr(x) _mm256_round_ps(x, 0x0)
401 #define GMX_SIMD_HAVE_FLOOR
402 #define gmx_floor_pr _mm256_floor_ps
404 #define GMX_SIMD_HAVE_BLENDV
405 #define gmx_blendv_pr _mm256_blendv_ps
407 static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
409 gmx_mm_pr sign_mask = _mm256_set1_ps(-0.0);
410 return _mm256_or_ps(_mm256_and_ps(a, sign_mask), b);
413 static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
415 return _mm256_add_ps(b, _mm256_andnot_ps(a, c));
418 /* Less-than (we use ordered, non-signaling, but that's not required) */
419 #define gmx_cmplt_pr(x, y) _mm256_cmp_ps(x, y, 0x11)
420 #define gmx_and_pb _mm256_and_ps
421 #define gmx_or_pb _mm256_or_ps
423 #define gmx_anytrue_pb _mm256_movemask_ps
425 #define gmx_cvttpr_epi32 _mm256_cvttps_epi32
427 #define gmx_rsqrt_pr _mm256_rsqrt_ps
428 #define gmx_rcp_pr _mm256_rcp_ps
430 #define gmx_exp_pr gmx_mm256_exp_ps
431 #define gmx_sqrt_pr gmx_mm256_sqrt_ps
432 #define gmx_sincos_pr gmx_mm256_sincos_ps
433 #define gmx_acos_pr gmx_mm256_acos_ps
434 #define gmx_atan2_pr gmx_mm256_atan2_ps
436 #else /* ifndef GMX_DOUBLE */
438 #define GMX_SIMD_WIDTH_HERE 4
440 #define gmx_mm_pr __m256d
442 #define gmx_mm_pb __m256d
444 /* We use 128-bit integer registers because of missing 256-bit operations */
445 #define gmx_epi32 __m128i
446 #define GMX_SIMD_EPI32_WIDTH 4
448 #define gmx_load_pr _mm256_load_pd
449 #define gmx_load1_pr(x) _mm256_set1_pd((x)[0])
450 #define gmx_set1_pr _mm256_set1_pd
451 #define gmx_setzero_pr _mm256_setzero_pd
452 #define gmx_store_pr _mm256_store_pd
454 #define gmx_add_pr _mm256_add_pd
455 #define gmx_sub_pr _mm256_sub_pd
456 #define gmx_mul_pr _mm256_mul_pd
457 #define gmx_madd_pr(a, b, c) _mm256_add_pd(c, _mm256_mul_pd(a, b))
458 #define gmx_nmsub_pr(a, b, c) _mm256_sub_pd(c, _mm256_mul_pd(a, b))
459 #define gmx_max_pr _mm256_max_pd
460 #define gmx_blendzero_pr _mm256_and_pd
462 #define gmx_round_pr(x) _mm256_round_pd(x, 0x0)
463 #define GMX_SIMD_HAVE_FLOOR
464 #define gmx_floor_pr _mm256_floor_pd
466 #define GMX_SIMD_HAVE_BLENDV
467 #define gmx_blendv_pr _mm256_blendv_pd
469 static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
471 gmx_mm_pr sign_mask = _mm256_set1_pd(-0.0);
472 return _mm256_or_pd(_mm256_and_pd(a, sign_mask), b);
475 static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
477 return _mm256_add_pd(b, _mm256_andnot_pd(a, c));
480 /* Less-than (we use ordered, non-signaling, but that's not required) */
481 #define gmx_cmplt_pr(x, y) _mm256_cmp_pd(x, y, 0x11)
483 #define gmx_and_pb _mm256_and_pd
484 #define gmx_or_pb _mm256_or_pd
486 #define gmx_anytrue_pb _mm256_movemask_pd
488 #define gmx_cvttpr_epi32 _mm256_cvttpd_epi32
490 #define gmx_rsqrt_pr(r) _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(r)))
491 #define gmx_rcp_pr(r) _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(r)))
493 #define gmx_exp_pr gmx_mm256_exp_pd
494 #define gmx_sqrt_pr gmx_mm256_sqrt_pd
495 #define gmx_sincos_pr gmx_mm256_sincos_pd
496 #define gmx_acos_pr gmx_mm256_acos_pd
497 #define gmx_atan2_pr gmx_mm256_atan2_pd
499 #endif /* ifndef GMX_DOUBLE */
501 #endif /* 128- or 256-bit x86 SIMD */
503 #endif /* GMX_X86_SSE2 */
505 #endif /* GMX_TARGET_X86 */
507 #ifdef GMX_CPU_ACCELERATION_IBM_QPX
509 /* This hack works on the compilers that can reach this code. A real
510 solution with broader scope will be proposed in master branch. */
511 #define gmx_always_inline __attribute__((always_inline))
513 /* This is for the A2 core on BlueGene/Q that supports IBM's QPX
514 vector built-in functions */
515 #define GMX_HAVE_SIMD_MACROS
519 #include <mass_simd.h>
522 /* No need to version the code by the precision, because the QPX AXU
523 extends to and truncates from double precision for free. */
525 #define GMX_SIMD_WIDTH_HERE 4
526 typedef vector4double gmx_mm_pr;
527 typedef vector4double gmx_mm_pb;
528 typedef vector4double gmx_epi32;
529 #define GMX_SIMD_EPI32_WIDTH 4
531 static gmx_inline gmx_mm_pr gmx_always_inline gmx_load_pr(const real *a)
534 return vec_ld(0, (real *) a);
536 return vec_lda(0, (real *) a);
540 static gmx_inline gmx_mm_pr gmx_always_inline gmx_load1_pr(const real *a)
542 return vec_splats(*a);
545 static gmx_inline gmx_mm_pr gmx_always_inline gmx_set1_pr(real a)
547 return vec_splats(a);
550 static gmx_inline gmx_mm_pr gmx_always_inline gmx_setzero_pr()
552 return vec_splats(0.0);
555 static gmx_inline void gmx_always_inline gmx_store_pr(real *a, gmx_mm_pr b)
564 static gmx_inline gmx_mm_pr gmx_always_inline gmx_add_pr(gmx_mm_pr a, gmx_mm_pr b)
566 return vec_add(a, b);
569 static gmx_inline gmx_mm_pr gmx_always_inline gmx_sub_pr(gmx_mm_pr a, gmx_mm_pr b)
571 return vec_sub(a, b);
574 static gmx_inline gmx_mm_pr gmx_always_inline gmx_mul_pr(gmx_mm_pr a, gmx_mm_pr b)
576 return vec_mul(a, b);
579 static gmx_inline gmx_mm_pr gmx_always_inline gmx_madd_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
581 return vec_madd(a, b, c);
584 static gmx_inline gmx_mm_pr gmx_always_inline gmx_nmsub_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
586 return vec_nmsub(a, b, c);
589 static gmx_inline gmx_mm_pr gmx_always_inline gmx_max_pr(gmx_mm_pr a, gmx_mm_pr b)
591 return vec_sel(b, a, vec_sub(a, b));
594 static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendzero_pr(gmx_mm_pr a, gmx_mm_pr b)
596 return vec_sel(gmx_setzero_pr(), a, b);
599 static gmx_inline gmx_mm_pb gmx_always_inline gmx_cmplt_pr(gmx_mm_pr a, gmx_mm_pr b)
601 return vec_cmplt(a, b);
604 static gmx_inline gmx_mm_pb gmx_always_inline gmx_and_pb(gmx_mm_pb a, gmx_mm_pb b)
606 return vec_and(a, b);
609 static gmx_inline gmx_mm_pb gmx_always_inline gmx_or_pb(gmx_mm_pb a, gmx_mm_pb b)
614 static gmx_inline gmx_mm_pr gmx_always_inline gmx_round_pr(gmx_mm_pr a)
619 #define GMX_SIMD_HAVE_FLOOR
620 static gmx_inline gmx_mm_pr gmx_always_inline gmx_floor_pr(gmx_mm_pr a)
625 #define GMX_SIMD_HAVE_BLENDV
626 static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendv_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
628 return vec_sel(b, a, gmx_cmplt_pr(gmx_setzero_pr(), c));
631 static gmx_inline gmx_mm_pr gmx_always_inline gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
633 return vec_cpsgn(a, b);
636 static gmx_inline gmx_mm_pr gmx_always_inline gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
638 return vec_add(b, vec_sel(c, gmx_setzero_pr(), a));
641 static gmx_inline gmx_bool gmx_always_inline
642 GMX_SIMD_IS_TRUE(real x)
647 static gmx_inline gmx_epi32 gmx_always_inline gmx_cvttpr_epi32(gmx_mm_pr a)
649 return vec_ctiwuz(a);
651 /* Don't want this, we have floor */
652 /* #define gmx_cvtepi32_pr vec_cvtepi32 */
654 /* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
655 Architecture only promises 2^-8. So probably no need for
656 Newton-Raphson iterates at single or double. */
657 static gmx_inline gmx_mm_pr gmx_always_inline gmx_rsqrt_pr(gmx_mm_pr a)
659 return vec_rsqrte(a);
662 /* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
663 Architecture only promises 2^-5. So probably no need for
664 Newton-Raphson iterates at single or double. */
665 static gmx_inline gmx_mm_pr gmx_always_inline gmx_rcp_pr(gmx_mm_pr a)
670 /* Note that here, and below, we use the built-in SLEEF port when
671 compiling on BlueGene/Q with clang */
673 #define GMX_SIMD_HAVE_EXP
674 static gmx_inline gmx_mm_pr gmx_always_inline gmx_exp_pr(gmx_mm_pr a)
691 static gmx_inline gmx_mm_pr gmx_always_inline gmx_sqrt_pr(gmx_mm_pr a)
694 return vec_swsqrt_nochk(a);
696 return vec_swsqrt(a);
700 #define GMX_SIMD_HAVE_TRIGONOMETRIC
701 static gmx_inline int gmx_always_inline gmx_sincos_pr(gmx_mm_pr a, gmx_mm_pr *b, gmx_mm_pr *c)
719 static gmx_inline gmx_mm_pr gmx_always_inline gmx_acos_pr(gmx_mm_pr a)
736 /* NB The order of parameters here is correct; the
737 documentation of atan2[df]4 in SIMD MASS is wrong. */
738 static gmx_inline gmx_mm_pr gmx_always_inline gmx_atan2_pr(gmx_mm_pr a, gmx_mm_pr b)
742 return xatan2f(a, b);
748 return atan2f4(a, b);
750 return atan2d4(a, b);
755 static gmx_inline int gmx_always_inline
756 gmx_anytrue_pb(gmx_mm_pb a)
758 /* The "anytrue" is done solely on the QPX AXU (which is the only
759 available FPU). This is awkward, because pretty much no
760 "horizontal" SIMD-vector operations exist, unlike x86 where
761 SSE4.1 added various kinds of horizontal operations. So we have
762 to make do with shifting vector elements and operating on the
763 results. This makes for lots of data dependency, but the main
764 alternative of storing to memory and reloading is not going to
765 help, either. OpenMP over 2 or 4 hardware threads per core will
766 hide much of the latency from the data dependency. The
767 vec_extract() lets the compiler correctly use a floating-point
768 comparison on the zeroth vector element, which avoids needing
771 gmx_mm_pb vec_shifted_left_0 = a;
772 gmx_mm_pb vec_shifted_left_1 = vec_sldw(a, a, 1);
773 gmx_mm_pb vec_shifted_left_2 = vec_sldw(a, a, 2);
774 gmx_mm_pb vec_shifted_left_3 = vec_sldw(a, a, 3);
776 gmx_mm_pb vec_return = vec_or(vec_or(vec_shifted_left_2, vec_shifted_left_3),
777 vec_or(vec_shifted_left_0, vec_shifted_left_1));
778 return (0.0 < vec_extract(vec_return, 0));
781 #undef gmx_always_inline
783 #endif /* GMX_CPU_ACCELERATION_IBM_QPX */
785 #ifdef GMX_HAVE_SIMD_MACROS
786 /* Generic functions to extract a SIMD aligned pointer from a pointer x.
787 * x should have at least GMX_SIMD_WIDTH_HERE elements extra compared
788 * to how many you want to use, to avoid indexing outside the aligned region.
791 static gmx_inline real *
792 gmx_simd_align_real(const real *x)
794 return (real *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(real)-1))));
797 static gmx_inline int *
798 gmx_simd_align_int(const int *x)
800 return (int *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(int )-1))));
804 /* Include the math functions which only need the above macros,
805 * generally these are the ones that don't need masking operations.
808 #include "gmx_simd_math_double.h"
810 #include "gmx_simd_math_single.h"
814 #endif /* GMX_HAVE_SIMD_MACROS */
816 #endif /* _gmx_simd_macros_h_ */