2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2012, The GROMACS Development Team
6 * Copyright (c) 2012,2013, by the GROMACS development team, led by
7 * David van der Spoel, Berk Hess, Erik Lindahl, and including many
8 * others, as listed in the AUTHORS file in the top-level source
9 * directory and at http://www.gromacs.org.
11 * GROMACS is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * GROMACS is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with GROMACS; if not, see
23 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
24 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 * If you want to redistribute modifications to GROMACS, please
27 * consider that scientific software is very special. Version
28 * control is crucial - bugs must be traceable. We will be happy to
29 * consider code for inclusion in the official distribution, but
30 * derived work must not be called official GROMACS. Details are found
31 * in the README & COPYING files - if they are missing, get the
32 * official version at http://www.gromacs.org.
34 * To help us fund GROMACS development, we humbly ask that you cite
35 * the research papers on the package. Check out http://www.gromacs.org.
37 #ifndef _nbnxn_kernel_simd_utils_x86_128s_h_
38 #define _nbnxn_kernel_simd_utils_x86_128s_h_
40 /* This files contains all functions/macros for the SIMD kernels
41 * which have explicit dependencies on the j-cluster size and/or SIMD-width.
42 * The functionality which depends on the j-cluster size is:
45 * energy group pair energy storage
48 /* Collect element 0 and 1 of the 4 inputs to out0 and out1, respectively */
49 static gmx_inline void
50 gmx_shuffle_4_ps_fil01_to_2_ps(__m128 in0, __m128 in1, __m128 in2, __m128 in3,
51 __m128 *out0, __m128 *out1)
55 _c01 = _mm_movelh_ps(in0, in1);
56 _c23 = _mm_movelh_ps(in2, in3);
57 *out0 = _mm_shuffle_ps(_c01, _c23, _MM_SHUFFLE(2, 0, 2, 0));
58 *out1 = _mm_shuffle_ps(_c01, _c23, _MM_SHUFFLE(3, 1, 3, 1));
61 /* Collect element 2 of the 4 inputs to out */
62 static gmx_inline __m128
63 gmx_shuffle_4_ps_fil2_to_1_ps(__m128 in0, __m128 in1, __m128 in2, __m128 in3)
67 _c01 = _mm_shuffle_ps(in0, in1, _MM_SHUFFLE(3, 2, 3, 2));
68 _c23 = _mm_shuffle_ps(in2, in3, _MM_SHUFFLE(3, 2, 3, 2));
70 return _mm_shuffle_ps(_c01, _c23, _MM_SHUFFLE(2, 0, 2, 0));
73 /* Sum the elements within each input register and store the sums in out */
74 static gmx_inline __m128
75 gmx_mm_transpose_sum4_pr(__m128 in0, __m128 in1,
76 __m128 in2, __m128 in3)
78 _MM_TRANSPOSE4_PS(in0, in1, in2, in3);
79 in0 = _mm_add_ps(in0, in1);
80 in2 = _mm_add_ps(in2, in3);
82 return _mm_add_ps(in0, in2);
85 static gmx_inline void
86 load_lj_pair_params(const real *nbfp, const int *type, int aj,
87 __m128 *c6_S, __m128 *c12_S)
89 __m128 clj_S[UNROLLJ];
92 for (p = 0; p < UNROLLJ; p++)
94 /* Here we load 4 aligned floats, but we need just 2 */
95 clj_S[p] = gmx_load_pr(nbfp+type[aj+p]*NBFP_STRIDE);
97 gmx_shuffle_4_ps_fil01_to_2_ps(clj_S[0], clj_S[1], clj_S[2], clj_S[3], c6_S, c12_S);
100 /* The load_table functions below are performance critical.
101 * The routines issue UNROLLI*UNROLLJ _mm_load_ps calls.
102 * As these all have latencies, scheduling is crucial.
103 * The Intel compilers and CPUs seem to do a good job at this.
104 * But AMD CPUs perform significantly worse with gcc than with icc.
105 * Performance is improved a bit by using the extract function UNROLLJ times,
106 * instead of doing an _mm_store_si128 for every i-particle.
107 * This is only faster when we use FDV0 formatted tables, where we also need
108 * to multiple the index by 4, which can be done by a SIMD bit shift.
109 * With single precision AVX, 8 extracts are much slower than 1 store.
110 * Because of this, the load_table_f macro always takes the ti parameter,
111 * but it is only used with AVX.
114 static gmx_inline void
115 load_table_f(const real *tab_coul_FDV0, gmx_epi32 ti_S, int *ti,
116 __m128 *ctab0_S, __m128 *ctab1_S)
121 /* Table has 4 entries, left-shift index by 2 */
122 ti_S = _mm_slli_epi32(ti_S, 2);
123 /* Without SSE4.1 the extract macro needs an immediate: unroll */
124 idx[0] = gmx_mm_extract_epi32(ti_S, 0);
125 ctab_S[0] = _mm_load_ps(tab_coul_FDV0+idx[0]);
126 idx[1] = gmx_mm_extract_epi32(ti_S, 1);
127 ctab_S[1] = _mm_load_ps(tab_coul_FDV0+idx[1]);
128 idx[2] = gmx_mm_extract_epi32(ti_S, 2);
129 ctab_S[2] = _mm_load_ps(tab_coul_FDV0+idx[2]);
130 idx[3] = gmx_mm_extract_epi32(ti_S, 3);
131 ctab_S[3] = _mm_load_ps(tab_coul_FDV0+idx[3]);
133 /* Shuffle the force table entries to a convenient order */
134 gmx_shuffle_4_ps_fil01_to_2_ps(ctab_S[0], ctab_S[1], ctab_S[2], ctab_S[3], ctab0_S, ctab1_S);
137 static gmx_inline void
138 load_table_f_v(const real *tab_coul_FDV0, gmx_epi32 ti_S, int *ti,
139 __m128 *ctab0_S, __m128 *ctab1_S, __m128 *ctabv_S)
144 /* Table has 4 entries, left-shift index by 2 */
145 ti_S = _mm_slli_epi32(ti_S, 2);
146 /* Without SSE4.1 the extract macro needs an immediate: unroll */
147 idx[0] = gmx_mm_extract_epi32(ti_S, 0);
148 ctab_S[0] = _mm_load_ps(tab_coul_FDV0+idx[0]);
149 idx[1] = gmx_mm_extract_epi32(ti_S, 1);
150 ctab_S[1] = _mm_load_ps(tab_coul_FDV0+idx[1]);
151 idx[2] = gmx_mm_extract_epi32(ti_S, 2);
152 ctab_S[2] = _mm_load_ps(tab_coul_FDV0+idx[2]);
153 idx[3] = gmx_mm_extract_epi32(ti_S, 3);
154 ctab_S[3] = _mm_load_ps(tab_coul_FDV0+idx[3]);
156 /* Shuffle the force table entries to a convenient order */
157 gmx_shuffle_4_ps_fil01_to_2_ps(ctab_S[0], ctab_S[1], ctab_S[2], ctab_S[3], ctab0_S, ctab1_S);
159 *ctabv_S = gmx_shuffle_4_ps_fil2_to_1_ps(ctab_S[0], ctab_S[1], ctab_S[2], ctab_S[3]);
162 #endif /* _nbnxn_kernel_simd_utils_x86_s128s_h_ */