2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 #ifndef _nbnxn_kernel_simd_utils_x86_256d_h_
36 #define _nbnxn_kernel_simd_utils_x86_256d_h_
38 /* This files contains all functions/macros for the SIMD kernels
39 * which have explicit dependencies on the j-cluster size and/or SIMD-width.
40 * The functionality which depends on the j-cluster size is:
43 * energy group pair energy storage
46 typedef gmx_simd_real_t gmx_exclfilter;
47 static const int filter_stride = 2;
49 /* Transpose 2 double precision registers */
50 static gmx_inline void
51 gmx_mm_transpose2_op_pd(__m128d in0, __m128d in1,
52 __m128d *out0, __m128d *out1)
54 *out0 = _mm_unpacklo_pd(in0, in1);
55 *out1 = _mm_unpackhi_pd(in0, in1);
58 /* Sum the elements within each input register and store the sums in out */
59 static gmx_inline __m256d
60 gmx_mm_transpose_sum4_pr(__m256d in0, __m256d in1,
61 __m256d in2, __m256d in3)
63 in0 = _mm256_hadd_pd(in0, in1);
64 in2 = _mm256_hadd_pd(in2, in3);
66 return _mm256_add_pd(_mm256_permute2f128_pd(in0, in2, 0x20), _mm256_permute2f128_pd(in0, in2, 0x31));
69 static gmx_inline __m256
70 gmx_mm256_invsqrt_ps_single(__m256 x)
72 const __m256 half = _mm256_set_ps(0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5);
73 const __m256 three = _mm256_set_ps(3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0);
75 __m256 lu = _mm256_rsqrt_ps(x);
77 return _mm256_mul_ps(half, _mm256_mul_ps(_mm256_sub_ps(three, _mm256_mul_ps(_mm256_mul_ps(lu, lu), x)), lu));
80 /* Put two 128-bit 4-float registers into one 256-bit 8-float register */
81 static gmx_inline __m256
82 gmx_2_m128_to_m256(__m128 in0, __m128 in1)
84 return _mm256_insertf128_ps(_mm256_castps128_ps256(in0), in1, 1);
87 /* Put two 128-bit 2-double registers into one 256-bit 4-double register */
88 static gmx_inline __m256d
89 gmx_2_m128d_to_m256d(__m128d in0, __m128d in1)
91 return _mm256_insertf128_pd(_mm256_castpd128_pd256(in0), in1, 1);
94 /* Do 2 double precision invsqrt operations.
95 * Doing the SIMD rsqrt and the first Newton Raphson iteration
96 * in single precision gives full double precision accuracy.
98 static gmx_inline void
99 gmx_mm_invsqrt2_pd(__m256d in0, __m256d in1,
100 __m256d *out0, __m256d *out1)
102 const __m256d half = _mm256_set1_pd(0.5);
103 const __m256d three = _mm256_set1_pd(3.0);
107 s = gmx_2_m128_to_m256(_mm256_cvtpd_ps(in0), _mm256_cvtpd_ps(in1));
108 ir = gmx_mm256_invsqrt_ps_single(s);
109 lu0 = _mm256_cvtps_pd(_mm256_castps256_ps128(ir));
110 lu1 = _mm256_cvtps_pd(_mm256_extractf128_ps(ir, 1));
111 *out0 = _mm256_mul_pd(half, _mm256_mul_pd(_mm256_sub_pd(three, _mm256_mul_pd(_mm256_mul_pd(lu0, lu0), in0)), lu0));
112 *out1 = _mm256_mul_pd(half, _mm256_mul_pd(_mm256_sub_pd(three, _mm256_mul_pd(_mm256_mul_pd(lu1, lu1), in1)), lu1));
115 static gmx_inline void
116 load_lj_pair_params(const real *nbfp, const int *type, int aj,
117 __m256d *c6_S, __m256d *c12_S)
119 __m128d clj_S[UNROLLJ], c6t_S[2], c12t_S[2];
122 for (p = 0; p < UNROLLJ; p++)
124 clj_S[p] = _mm_load_pd(nbfp+type[aj+p]*nbfp_stride);
126 gmx_mm_transpose2_op_pd(clj_S[0], clj_S[1], &c6t_S[0], &c12t_S[0]);
127 gmx_mm_transpose2_op_pd(clj_S[2], clj_S[3], &c6t_S[1], &c12t_S[1]);
128 *c6_S = gmx_2_m128d_to_m256d(c6t_S[0], c6t_S[1]);
129 *c12_S = gmx_2_m128d_to_m256d(c12t_S[0], c12t_S[1]);
132 /* The load_table functions below are performance critical. They
133 * always take the ti parameter, which should contain a buffer that
134 * is aligned with prepare_table_load_buffer(), but it is only used
135 * with full-width AVX_256. */
137 static gmx_inline void
138 load_table_f(const real *tab_coul_F, __m128i ti_S, int *ti,
139 __m256d *ctab0_S, __m256d *ctab1_S)
141 __m128d ctab_S[4], tr_S[4];
144 _mm_store_si128((__m128i *)ti, ti_S);
145 for (j = 0; j < 4; j++)
147 ctab_S[j] = _mm_loadu_pd(tab_coul_F+ti[j]);
149 /* Shuffle the force table entries to a convenient order */
150 gmx_mm_transpose2_op_pd(ctab_S[0], ctab_S[1], &tr_S[0], &tr_S[1]);
151 gmx_mm_transpose2_op_pd(ctab_S[2], ctab_S[3], &tr_S[2], &tr_S[3]);
152 *ctab0_S = gmx_2_m128d_to_m256d(tr_S[0], tr_S[2]);
153 *ctab1_S = gmx_2_m128d_to_m256d(tr_S[1], tr_S[3]);
154 /* The second force table entry should contain the difference */
155 *ctab1_S = _mm256_sub_pd(*ctab1_S, *ctab0_S);
158 static gmx_inline void
159 load_table_f_v(const real *tab_coul_F, const real *tab_coul_V,
160 __m128i ti_S, int *ti,
161 __m256d *ctab0_S, __m256d *ctab1_S, __m256d *ctabv_S)
163 __m128d ctab_S[8], tr_S[4];
166 _mm_store_si128((__m128i *)ti, ti_S);
167 for (j = 0; j < 4; j++)
169 ctab_S[j] = _mm_loadu_pd(tab_coul_F+ti[j]);
171 /* Shuffle the force table entries to a convenient order */
172 gmx_mm_transpose2_op_pd(ctab_S[0], ctab_S[1], &tr_S[0], &tr_S[1]);
173 gmx_mm_transpose2_op_pd(ctab_S[2], ctab_S[3], &tr_S[2], &tr_S[3]);
174 *ctab0_S = gmx_2_m128d_to_m256d(tr_S[0], tr_S[2]);
175 *ctab1_S = gmx_2_m128d_to_m256d(tr_S[1], tr_S[3]);
176 /* The second force table entry should contain the difference */
177 *ctab1_S = _mm256_sub_pd(*ctab1_S, *ctab0_S);
179 for (j = 0; j < 4; j++)
181 ctab_S[4+j] = _mm_loadu_pd(tab_coul_V+ti[j]);
183 /* Shuffle the energy table entries to a single register */
184 *ctabv_S = gmx_2_m128d_to_m256d(_mm_shuffle_pd(ctab_S[4], ctab_S[5], _MM_SHUFFLE2(0, 0)), _mm_shuffle_pd(ctab_S[6], ctab_S[7], _MM_SHUFFLE2(0, 0)));
187 static gmx_inline gmx_exclfilter
188 gmx_load1_exclfilter(int e)
190 return _mm256_castsi256_pd(_mm256_set1_epi32(e));
193 static gmx_inline gmx_exclfilter
194 gmx_load_exclusion_filter(const unsigned *i)
196 return gmx_simd_load_r((real *) (i));
199 static gmx_inline gmx_simd_bool_t
200 gmx_checkbitmask_pb(gmx_exclfilter m0, gmx_exclfilter m1)
202 /* With <= 16 bits used the cast and conversion should not be
203 * required, since only mantissa bits are set and that would give
204 * a non-zero float, but with the Intel compiler this does not
205 * work correctly. Because AVX does not have int->double
206 * conversion, we convert via float. */
207 return _mm256_cmp_pd(_mm256_castps_pd(_mm256_cvtepi32_ps(_mm256_castpd_si256(_mm256_and_pd(m0, m1)))), _mm256_setzero_pd(), 0x0c);
210 #endif /* _nbnxn_kernel_simd_utils_x86_s256d_h_ */