2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2014,2015,2016,2017,2018 by the GROMACS development team.
5 * Copyright (c) 2019,2020, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
37 #ifndef GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H
38 #define GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H
45 #include <immintrin.h>
47 #include "gromacs/math/utilities.h"
48 #include "gromacs/utility/real.h"
50 #include "impl_x86_avx_512_general.h"
60 SimdFloat(float f) : simdInternal_(_mm512_set1_ps(f)) {}
62 // Internal utility constructor to simplify return statements
63 SimdFloat(__m512 simd) : simdInternal_(simd) {}
73 SimdFInt32(std::int32_t i) : simdInternal_(_mm512_set1_epi32(i)) {}
75 // Internal utility constructor to simplify return statements
76 SimdFInt32(__m512i simd) : simdInternal_(simd) {}
78 __m512i simdInternal_;
86 // Internal utility constructor to simplify return statements
87 SimdFBool(__mmask16 simd) : simdInternal_(simd) {}
89 __mmask16 simdInternal_;
97 // Internal utility constructor to simplify return statements
98 SimdFIBool(__mmask16 simd) : simdInternal_(simd) {}
100 __mmask16 simdInternal_;
103 static inline SimdFloat gmx_simdcall simdLoad(const float* m, SimdFloatTag = {})
105 assert(std::size_t(m) % 64 == 0);
106 return { _mm512_load_ps(m) };
109 static inline void gmx_simdcall store(float* m, SimdFloat a)
111 assert(std::size_t(m) % 64 == 0);
112 _mm512_store_ps(m, a.simdInternal_);
115 static inline SimdFloat gmx_simdcall simdLoadU(const float* m, SimdFloatTag = {})
117 return { _mm512_loadu_ps(m) };
120 static inline void gmx_simdcall storeU(float* m, SimdFloat a)
122 _mm512_storeu_ps(m, a.simdInternal_);
125 static inline SimdFloat gmx_simdcall setZeroF()
127 return { _mm512_setzero_ps() };
130 static inline SimdFInt32 gmx_simdcall simdLoad(const std::int32_t* m, SimdFInt32Tag)
132 assert(std::size_t(m) % 64 == 0);
133 return { _mm512_load_si512(m) };
136 static inline void gmx_simdcall store(std::int32_t* m, SimdFInt32 a)
138 assert(std::size_t(m) % 64 == 0);
139 _mm512_store_si512(m, a.simdInternal_);
142 static inline SimdFInt32 gmx_simdcall simdLoadU(const std::int32_t* m, SimdFInt32Tag)
144 return { _mm512_loadu_si512(m) };
147 static inline void gmx_simdcall storeU(std::int32_t* m, SimdFInt32 a)
149 _mm512_storeu_si512(m, a.simdInternal_);
152 static inline SimdFInt32 gmx_simdcall setZeroFI()
154 return { _mm512_setzero_si512() };
158 static inline SimdFloat gmx_simdcall operator&(SimdFloat a, SimdFloat b)
160 return { _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a.simdInternal_),
161 _mm512_castps_si512(b.simdInternal_))) };
164 static inline SimdFloat gmx_simdcall andNot(SimdFloat a, SimdFloat b)
166 return { _mm512_castsi512_ps(_mm512_andnot_epi32(_mm512_castps_si512(a.simdInternal_),
167 _mm512_castps_si512(b.simdInternal_))) };
170 static inline SimdFloat gmx_simdcall operator|(SimdFloat a, SimdFloat b)
172 return { _mm512_castsi512_ps(_mm512_or_epi32(_mm512_castps_si512(a.simdInternal_),
173 _mm512_castps_si512(b.simdInternal_))) };
176 static inline SimdFloat gmx_simdcall operator^(SimdFloat a, SimdFloat b)
178 return { _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a.simdInternal_),
179 _mm512_castps_si512(b.simdInternal_))) };
182 static inline SimdFloat gmx_simdcall operator+(SimdFloat a, SimdFloat b)
184 return { _mm512_add_ps(a.simdInternal_, b.simdInternal_) };
187 static inline SimdFloat gmx_simdcall operator-(SimdFloat a, SimdFloat b)
189 return { _mm512_sub_ps(a.simdInternal_, b.simdInternal_) };
192 static inline SimdFloat gmx_simdcall operator-(SimdFloat x)
194 return { _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(x.simdInternal_),
195 _mm512_castps_si512(_mm512_set1_ps(GMX_FLOAT_NEGZERO)))) };
198 static inline SimdFloat gmx_simdcall operator*(SimdFloat a, SimdFloat b)
200 return { _mm512_mul_ps(a.simdInternal_, b.simdInternal_) };
203 static inline SimdFloat gmx_simdcall fma(SimdFloat a, SimdFloat b, SimdFloat c)
205 return { _mm512_fmadd_ps(a.simdInternal_, b.simdInternal_, c.simdInternal_) };
208 static inline SimdFloat gmx_simdcall fms(SimdFloat a, SimdFloat b, SimdFloat c)
210 return { _mm512_fmsub_ps(a.simdInternal_, b.simdInternal_, c.simdInternal_) };
213 static inline SimdFloat gmx_simdcall fnma(SimdFloat a, SimdFloat b, SimdFloat c)
215 return { _mm512_fnmadd_ps(a.simdInternal_, b.simdInternal_, c.simdInternal_) };
218 static inline SimdFloat gmx_simdcall fnms(SimdFloat a, SimdFloat b, SimdFloat c)
220 return { _mm512_fnmsub_ps(a.simdInternal_, b.simdInternal_, c.simdInternal_) };
223 // Override for AVX-512-KNL
224 #if GMX_SIMD_X86_AVX_512
225 static inline SimdFloat gmx_simdcall rsqrt(SimdFloat x)
227 return { _mm512_rsqrt14_ps(x.simdInternal_) };
230 static inline SimdFloat gmx_simdcall rcp(SimdFloat x)
232 return { _mm512_rcp14_ps(x.simdInternal_) };
236 static inline SimdFloat gmx_simdcall maskAdd(SimdFloat a, SimdFloat b, SimdFBool m)
238 return { _mm512_mask_add_ps(a.simdInternal_, m.simdInternal_, a.simdInternal_, b.simdInternal_) };
241 static inline SimdFloat gmx_simdcall maskzMul(SimdFloat a, SimdFloat b, SimdFBool m)
243 return { _mm512_maskz_mul_ps(m.simdInternal_, a.simdInternal_, b.simdInternal_) };
246 static inline SimdFloat gmx_simdcall maskzFma(SimdFloat a, SimdFloat b, SimdFloat c, SimdFBool m)
248 return { _mm512_maskz_fmadd_ps(m.simdInternal_, a.simdInternal_, b.simdInternal_, c.simdInternal_) };
251 // Override for AVX-512-KNL
252 #if GMX_SIMD_X86_AVX_512
253 static inline SimdFloat gmx_simdcall maskzRsqrt(SimdFloat x, SimdFBool m)
255 return { _mm512_maskz_rsqrt14_ps(m.simdInternal_, x.simdInternal_) };
258 static inline SimdFloat gmx_simdcall maskzRcp(SimdFloat x, SimdFBool m)
260 return { _mm512_maskz_rcp14_ps(m.simdInternal_, x.simdInternal_) };
264 static inline SimdFloat gmx_simdcall abs(SimdFloat x)
266 return { _mm512_castsi512_ps(_mm512_andnot_epi32(_mm512_castps_si512(_mm512_set1_ps(GMX_FLOAT_NEGZERO)),
267 _mm512_castps_si512(x.simdInternal_))) };
270 static inline SimdFloat gmx_simdcall max(SimdFloat a, SimdFloat b)
272 return { _mm512_max_ps(a.simdInternal_, b.simdInternal_) };
275 static inline SimdFloat gmx_simdcall min(SimdFloat a, SimdFloat b)
277 return { _mm512_min_ps(a.simdInternal_, b.simdInternal_) };
280 static inline SimdFloat gmx_simdcall round(SimdFloat x)
282 return { _mm512_roundscale_ps(x.simdInternal_, 0) };
285 static inline SimdFloat gmx_simdcall trunc(SimdFloat x)
287 #if defined(__INTEL_COMPILER) || defined(__ECC)
288 return { _mm512_trunc_ps(x.simdInternal_) };
290 return { _mm512_cvtepi32_ps(_mm512_cvttps_epi32(x.simdInternal_)) };
294 static inline SimdFloat gmx_simdcall frexp(SimdFloat value, SimdFInt32* exponent)
296 __m512 rExponent = _mm512_getexp_ps(value.simdInternal_);
297 __m512i iExponent = _mm512_cvtps_epi32(rExponent);
299 exponent->simdInternal_ = _mm512_add_epi32(iExponent, _mm512_set1_epi32(1));
301 return { _mm512_getmant_ps(value.simdInternal_, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src) };
304 template<MathOptimization opt = MathOptimization::Safe>
305 static inline SimdFloat gmx_simdcall ldexp(SimdFloat value, SimdFInt32 exponent)
307 const __m512i exponentBias = _mm512_set1_epi32(127);
308 __m512i iExponent = _mm512_add_epi32(exponent.simdInternal_, exponentBias);
310 if (opt == MathOptimization::Safe)
312 // Make sure biased argument is not negative
313 iExponent = _mm512_max_epi32(iExponent, _mm512_setzero_epi32());
316 iExponent = _mm512_slli_epi32(iExponent, 23);
318 return { _mm512_mul_ps(value.simdInternal_, _mm512_castsi512_ps(iExponent)) };
321 static inline float gmx_simdcall reduce(SimdFloat a)
323 __m512 x = a.simdInternal_;
324 x = _mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, 0xEE));
325 x = _mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, 0x11));
326 x = _mm512_add_ps(x, _mm512_permute_ps(x, 0xEE));
327 x = _mm512_add_ps(x, _mm512_permute_ps(x, 0x11));
328 return *reinterpret_cast<float*>(&x);
331 static inline SimdFBool gmx_simdcall operator==(SimdFloat a, SimdFloat b)
333 return { _mm512_cmp_ps_mask(a.simdInternal_, b.simdInternal_, _CMP_EQ_OQ) };
336 static inline SimdFBool gmx_simdcall operator!=(SimdFloat a, SimdFloat b)
338 return { _mm512_cmp_ps_mask(a.simdInternal_, b.simdInternal_, _CMP_NEQ_OQ) };
341 static inline SimdFBool gmx_simdcall operator<(SimdFloat a, SimdFloat b)
343 return { _mm512_cmp_ps_mask(a.simdInternal_, b.simdInternal_, _CMP_LT_OQ) };
346 static inline SimdFBool gmx_simdcall operator<=(SimdFloat a, SimdFloat b)
348 return { _mm512_cmp_ps_mask(a.simdInternal_, b.simdInternal_, _CMP_LE_OQ) };
351 static inline SimdFBool gmx_simdcall testBits(SimdFloat a)
353 return { _mm512_test_epi32_mask(_mm512_castps_si512(a.simdInternal_),
354 _mm512_castps_si512(a.simdInternal_)) };
357 static inline SimdFBool gmx_simdcall operator&&(SimdFBool a, SimdFBool b)
359 return { _mm512_kand(a.simdInternal_, b.simdInternal_) };
362 static inline SimdFBool gmx_simdcall operator||(SimdFBool a, SimdFBool b)
364 return { _mm512_kor(a.simdInternal_, b.simdInternal_) };
367 static inline bool gmx_simdcall anyTrue(SimdFBool a)
369 return (avx512Mask2Int(a.simdInternal_) != 0);
372 static inline SimdFloat gmx_simdcall selectByMask(SimdFloat a, SimdFBool m)
374 return { _mm512_mask_mov_ps(_mm512_setzero_ps(), m.simdInternal_, a.simdInternal_) };
377 static inline SimdFloat gmx_simdcall selectByNotMask(SimdFloat a, SimdFBool m)
379 return { _mm512_mask_mov_ps(a.simdInternal_, m.simdInternal_, _mm512_setzero_ps()) };
382 static inline SimdFloat gmx_simdcall blend(SimdFloat a, SimdFloat b, SimdFBool sel)
384 return { _mm512_mask_blend_ps(sel.simdInternal_, a.simdInternal_, b.simdInternal_) };
387 static inline SimdFloat gmx_simdcall copysign(SimdFloat a, SimdFloat b)
389 return { _mm512_castsi512_ps(_mm512_ternarylogic_epi32(_mm512_castps_si512(a.simdInternal_),
390 _mm512_castps_si512(b.simdInternal_),
391 _mm512_set1_epi32(INT32_MIN), 0xD8)) };
394 static inline SimdFInt32 gmx_simdcall operator&(SimdFInt32 a, SimdFInt32 b)
396 return { _mm512_and_epi32(a.simdInternal_, b.simdInternal_) };
399 static inline SimdFInt32 gmx_simdcall andNot(SimdFInt32 a, SimdFInt32 b)
401 return { _mm512_andnot_epi32(a.simdInternal_, b.simdInternal_) };
404 static inline SimdFInt32 gmx_simdcall operator|(SimdFInt32 a, SimdFInt32 b)
406 return { _mm512_or_epi32(a.simdInternal_, b.simdInternal_) };
409 static inline SimdFInt32 gmx_simdcall operator^(SimdFInt32 a, SimdFInt32 b)
411 return { _mm512_xor_epi32(a.simdInternal_, b.simdInternal_) };
414 static inline SimdFInt32 gmx_simdcall operator+(SimdFInt32 a, SimdFInt32 b)
416 return { _mm512_add_epi32(a.simdInternal_, b.simdInternal_) };
419 static inline SimdFInt32 gmx_simdcall operator-(SimdFInt32 a, SimdFInt32 b)
421 return { _mm512_sub_epi32(a.simdInternal_, b.simdInternal_) };
424 static inline SimdFInt32 gmx_simdcall operator*(SimdFInt32 a, SimdFInt32 b)
426 return { _mm512_mullo_epi32(a.simdInternal_, b.simdInternal_) };
429 static inline SimdFIBool gmx_simdcall operator==(SimdFInt32 a, SimdFInt32 b)
431 return { _mm512_cmp_epi32_mask(a.simdInternal_, b.simdInternal_, _MM_CMPINT_EQ) };
434 static inline SimdFIBool gmx_simdcall testBits(SimdFInt32 a)
436 return { _mm512_test_epi32_mask(a.simdInternal_, a.simdInternal_) };
439 static inline SimdFIBool gmx_simdcall operator<(SimdFInt32 a, SimdFInt32 b)
441 return { _mm512_cmp_epi32_mask(a.simdInternal_, b.simdInternal_, _MM_CMPINT_LT) };
444 static inline SimdFIBool gmx_simdcall operator&&(SimdFIBool a, SimdFIBool b)
446 return { _mm512_kand(a.simdInternal_, b.simdInternal_) };
449 static inline SimdFIBool gmx_simdcall operator||(SimdFIBool a, SimdFIBool b)
451 return { _mm512_kor(a.simdInternal_, b.simdInternal_) };
454 static inline bool gmx_simdcall anyTrue(SimdFIBool a)
456 return (avx512Mask2Int(a.simdInternal_) != 0);
459 static inline SimdFInt32 gmx_simdcall selectByMask(SimdFInt32 a, SimdFIBool m)
461 return { _mm512_mask_mov_epi32(_mm512_setzero_epi32(), m.simdInternal_, a.simdInternal_) };
464 static inline SimdFInt32 gmx_simdcall selectByNotMask(SimdFInt32 a, SimdFIBool m)
466 return { _mm512_mask_mov_epi32(a.simdInternal_, m.simdInternal_, _mm512_setzero_epi32()) };
469 static inline SimdFInt32 gmx_simdcall blend(SimdFInt32 a, SimdFInt32 b, SimdFIBool sel)
471 return { _mm512_mask_blend_epi32(sel.simdInternal_, a.simdInternal_, b.simdInternal_) };
474 static inline SimdFInt32 gmx_simdcall cvtR2I(SimdFloat a)
476 return { _mm512_cvtps_epi32(a.simdInternal_) };
479 static inline SimdFInt32 gmx_simdcall cvttR2I(SimdFloat a)
481 return { _mm512_cvttps_epi32(a.simdInternal_) };
484 static inline SimdFloat gmx_simdcall cvtI2R(SimdFInt32 a)
486 return { _mm512_cvtepi32_ps(a.simdInternal_) };
489 static inline SimdFIBool gmx_simdcall cvtB2IB(SimdFBool a)
491 return { a.simdInternal_ };
494 static inline SimdFBool gmx_simdcall cvtIB2B(SimdFIBool a)
496 return { a.simdInternal_ };
501 #endif // GMX_SIMD_IMPL_X86_AVX_512_SIMD_FLOAT_H