2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 #ifndef GMX_SIMD_IMPL_X86_AVX2_256_H
37 #define GMX_SIMD_IMPL_X86_AVX2_256_H
40 #include <immintrin.h>
44 /* x86 256-bit AVX2 SIMD instruction wrappers
46 * Please see documentation in gromacs/simd/simd.h for details
49 /* Inherit parts of AVX2_256 from AVX_256 */
50 #include "gromacs/simd/impl_x86_avx_256/impl_x86_avx_256.h"
51 /* Increment over AVX_256 capabilities */
52 #define GMX_SIMD_X86_AVX2_256_OR_HIGHER
54 /* Override some capability definitions for things added in AVX2 */
55 #define GMX_SIMD_HAVE_FMA
56 #define GMX_SIMD_HAVE_FINT32_LOGICAL /* AVX2 adds 256-bit integer shifts */
57 #define GMX_SIMD_HAVE_FINT32_ARITHMETICS /* AVX2 adds 256-bit integer +,-,* */
59 /****************************************************
60 * SINGLE PRECISION SIMD IMPLEMENTATION *
61 ****************************************************/
62 #undef gmx_simd_fmadd_f
63 #define gmx_simd_fmadd_f _mm256_fmadd_ps
64 #undef gmx_simd_fmsub_f
65 #define gmx_simd_fmsub_f _mm256_fmsub_ps
66 #undef gmx_simd_fnmadd_f
67 #define gmx_simd_fnmadd_f _mm256_fnmadd_ps
68 #undef gmx_simd_fnmsub_f
69 #define gmx_simd_fnmsub_f _mm256_fnmsub_ps
70 #undef gmx_simd_get_exponent_f
71 #define gmx_simd_get_exponent_f gmx_simd_get_exponent_f_avx2_256
72 #undef gmx_simd_set_exponent_f
73 #define gmx_simd_set_exponent_f gmx_simd_set_exponent_f_avx2_256
74 /* Previously undefined logical ops on gmx_simd_fint32_t */
75 #define gmx_simd_slli_fi _mm256_slli_epi32
76 #define gmx_simd_srli_fi _mm256_srli_epi32
77 #define gmx_simd_and_fi _mm256_and_si256
78 #define gmx_simd_andnot_fi _mm256_andnot_si256
79 #define gmx_simd_or_fi _mm256_or_si256
80 #define gmx_simd_xor_fi _mm256_xor_si256
81 /* Previously undefined arithmetic ops on gmx_simd_fint32_t */
82 #define gmx_simd_add_fi _mm256_add_epi32
83 #define gmx_simd_sub_fi _mm256_sub_epi32
84 #define gmx_simd_mul_fi _mm256_mullo_epi32
85 /* Previously undefined boolean ops on gmx_simd_fint32_t */
86 #define gmx_simd_cmpeq_fi _mm256_cmpeq_epi32
87 #define gmx_simd_cmplt_fi(a, b) _mm256_cmpgt_epi32(b, a)
88 #define gmx_simd_and_fib _mm256_and_si256
89 #define gmx_simd_or_fib _mm256_or_si256
90 #define gmx_simd_anytrue_fib _mm256_movemask_epi8
91 #define gmx_simd_blendzero_fi _mm256_and_si256
92 #define gmx_simd_blendnotzero_fi(a, sel) _mm256_andnot_si256(sel, a)
93 #define gmx_simd_blendv_fi _mm256_blendv_epi8
96 /****************************************************
97 * DOUBLE PRECISION SIMD IMPLEMENTATION *
98 ****************************************************/
99 #undef gmx_simd_fmadd_d
100 #define gmx_simd_fmadd_d _mm256_fmadd_pd
101 #undef gmx_simd_fmsub_d
102 #define gmx_simd_fmsub_d _mm256_fmsub_pd
103 #undef gmx_simd_fnmadd_d
104 #define gmx_simd_fnmadd_d _mm256_fnmadd_pd
105 #undef gmx_simd_fnmsub_d
106 #define gmx_simd_fnmsub_d _mm256_fnmsub_pd
107 #undef gmx_simd_get_exponent_d
108 #define gmx_simd_get_exponent_d gmx_simd_get_exponent_d_avx2_256
109 #undef gmx_simd_set_exponent_d
110 #define gmx_simd_set_exponent_d gmx_simd_set_exponent_d_avx2_256
111 #undef gmx_simd_cvt_db2dib
112 #define gmx_simd_cvt_db2dib gmx_simd_cvt_db2dib_avx2_256
113 #undef gmx_simd_cvt_dib2db
114 #define gmx_simd_cvt_dib2db gmx_simd_cvt_dib2db_avx2_256
116 /****************************************************
117 * SIMD4 SINGLE PRECISION IMPLEMENTATION *
118 ****************************************************/
119 #undef gmx_simd4_fmadd_f
120 #define gmx_simd4_fmadd_f _mm_fmadd_ps
121 #undef gmx_simd4_fmsub_f
122 #define gmx_simd4_fmsub_f _mm_fmsub_ps
123 #undef gmx_simd4_fnmadd_f
124 #define gmx_simd4_fnmadd_f _mm_fnmadd_ps
125 #undef gmx_simd4_fnmsub_f
126 #define gmx_simd4_fnmsub_f _mm_fnmsub_ps
128 /* No need to update SIMD4 double, since those instructions
129 * are aliased to the general SIMD double instructions above.
132 /*********************************************************
133 * SIMD SINGLE PRECISION IMPLEMENTATION HELPER FUNCTIONS *
134 *********************************************************/
135 static gmx_inline gmx_simd_float_t gmx_simdcall
136 gmx_simd_get_exponent_f_avx2_256(gmx_simd_float_t x)
138 const __m256 expmask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7F800000));
139 const __m256i expbias = _mm256_set1_epi32(127);
142 iexp = _mm256_castps_si256(_mm256_and_ps(x, expmask));
143 iexp = _mm256_sub_epi32(_mm256_srli_epi32(iexp, 23), expbias);
144 return _mm256_cvtepi32_ps(iexp);
147 static gmx_inline gmx_simd_float_t gmx_simdcall
148 gmx_simd_set_exponent_f_avx2_256(gmx_simd_float_t x)
150 const __m256i expbias = _mm256_set1_epi32(127);
151 __m256i iexp = _mm256_cvtps_epi32(x);
153 iexp = _mm256_slli_epi32(_mm256_add_epi32(iexp, expbias), 23);
154 return _mm256_castsi256_ps(iexp);
157 /*********************************************************
158 * SIMD DOUBLE PRECISION IMPLEMENTATION HELPER FUNCTIONS *
159 *********************************************************/
160 static gmx_inline gmx_simd_double_t gmx_simdcall
161 gmx_simd_get_exponent_d_avx2_256(gmx_simd_double_t x)
163 const __m256d expmask = _mm256_castsi256_pd(_mm256_set1_epi64x(0x7FF0000000000000LL));
164 const __m256i expbias = _mm256_set1_epi64x(1023LL);
168 iexp = _mm256_castpd_si256(_mm256_and_pd(x, expmask));
169 iexp = _mm256_sub_epi64(_mm256_srli_epi64(iexp, 52), expbias);
170 iexp = _mm256_shuffle_epi32(iexp, _MM_SHUFFLE(3, 1, 2, 0));
172 iexp128 = _mm256_extractf128_si256(iexp, 1);
173 iexp128 = _mm_unpacklo_epi64(_mm256_castsi256_si128(iexp), iexp128);
174 return _mm256_cvtepi32_pd(iexp128);
177 static gmx_inline gmx_simd_double_t gmx_simdcall
178 gmx_simd_set_exponent_d_avx2_256(gmx_simd_double_t x)
180 const __m256i expbias = _mm256_set1_epi64x(1023LL);
181 __m256i iexp = _mm256_cvtepi32_epi64(_mm256_cvtpd_epi32(x));
183 iexp = _mm256_slli_epi64(_mm256_add_epi64(iexp, expbias), 52);
184 return _mm256_castsi256_pd(iexp);
187 static gmx_inline gmx_simd_dibool_t gmx_simdcall
188 gmx_simd_cvt_db2dib_avx2_256(gmx_simd_dbool_t a)
190 __m128i ia = _mm256_castsi256_si128(_mm256_castpd_si256(a));
191 __m128i ib = _mm256_extractf128_si256(_mm256_castpd_si256(a), 0x1);
193 ia = _mm_packs_epi32(ia, ib);
198 static gmx_inline gmx_simd_dbool_t gmx_simdcall
199 gmx_simd_cvt_dib2db_avx2_256(gmx_simd_dibool_t ia)
201 __m128d lo = _mm_castsi128_pd(_mm_unpacklo_epi32(ia, ia));
202 __m128d hi = _mm_castsi128_pd(_mm_unpackhi_epi32(ia, ia));
204 return _mm256_insertf128_pd(_mm256_castpd128_pd256(lo), hi, 0x1);
207 #endif /* GMX_SIMD_IMPL_X86_AVX2_256_H */