/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
- * Copyright (c) 2001-2012, The GROMACS Development Team
- * Copyright (c) 2012,2013, by the GROMACS development team, led by
- * David van der Spoel, Berk Hess, Erik Lindahl, and including many
- * others, as listed in the AUTHORS file in the top-level source
- * directory and at http://www.gromacs.org.
+ * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
*
* GROMACS is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
#ifndef _nbnxn_kernel_simd_utils_x86_256s_h_
#define _nbnxn_kernel_simd_utils_x86_256s_h_
+#include "config.h"
+
/* This files contains all functions/macros for the SIMD kernels
* which have explicit dependencies on the j-cluster size and/or SIMD-width.
* The functionality which depends on the j-cluster size is:
*/
-/* The 4xn kernel operates on 4-wide i-force registers */
-#define gmx_mm_pr4 __m128
-#define gmx_load_pr4 _mm_load_ps
-#define gmx_store_pr4 _mm_store_ps
-#define gmx_add_pr4 _mm_add_ps
-
-
+#ifdef GMX_NBNXN_SIMD_2XNN
/* Half-width operations are required for the 2xnn kernels */
/* Half-width SIMD real type */
#define gmx_set1_hpr(a, b) *(a) = _mm_set1_ps(b)
/* Load one real at b and one real at b+1 into halves of a, respectively */
#define gmx_load1p1_pr(a, b) *(a) = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm_load1_ps(b)), _mm_load1_ps(b+1), 0x1)
-/* Load reals at half-width aligned pointer b into two halves of a */
-#define gmx_loaddh_pr(a, b) *(a) = gmx_mm256_load4_ps(b)
/* To half-width SIMD register b into half width aligned memory a */
#define gmx_store_hpr(a, b) _mm_store_ps(a, b)
#define gmx_add_hpr _mm_add_ps
#define gmx_sub_hpr _mm_sub_ps
+
/* Sum over 4 half SIMD registers */
-#define gmx_sum4_hpr gmx_mm256_sum4h_m128
+static __m128 gmx_simdcall gmx_sum4_hpr(__m256 x, __m256 y)
+{
+ __m256 sum;
+ sum = _mm256_add_ps(x, y);
+ return _mm_add_ps(_mm256_castps256_ps128(sum), _mm256_extractf128_ps(sum, 0x1));
+}
+
+/* Load reals at half-width aligned pointer b into two halves of a */
static gmx_inline void
-gmx_pr_to_2hpr(gmx_mm_pr a, gmx_mm_hpr *b, gmx_mm_hpr *c)
+gmx_loaddh_pr(gmx_simd_real_t *a, const real *b)
+{
+ __m128 tmp;
+ tmp = _mm_load_ps(b);
+ *a = _mm256_insertf128_ps(_mm256_castps128_ps256(tmp), tmp, 0x1);
+}
+
+static gmx_inline void gmx_simdcall
+gmx_pr_to_2hpr(gmx_simd_real_t a, gmx_mm_hpr *b, gmx_mm_hpr *c)
{
*b = _mm256_extractf128_ps(a, 0);
*c = _mm256_extractf128_ps(a, 1);
}
/* Store half width SIMD registers a and b in full width register *c */
-static gmx_inline void
-gmx_2hpr_to_pr(gmx_mm_hpr a, gmx_mm_hpr b, gmx_mm_pr *c)
+static gmx_inline void gmx_simdcall
+gmx_2hpr_to_pr(gmx_mm_hpr a, gmx_mm_hpr b, gmx_simd_real_t *c)
{
*c = _mm256_insertf128_ps(_mm256_castps128_ps256(a), b, 0x1);
}
+#endif /* GMX_NBNXN_SIMD_2XNN */
+
/* Collect element 0 and 1 of the 4 inputs to out0 and out1, respectively */
-static gmx_inline void
+static gmx_inline void gmx_simdcall
gmx_shuffle_4_ps_fil01_to_2_ps(__m128 in0, __m128 in1, __m128 in2, __m128 in3,
__m128 *out0, __m128 *out1)
{
}
/* Collect element 2 of the 4 inputs to out */
-static gmx_inline __m128
+static gmx_inline __m128 gmx_simdcall
gmx_shuffle_4_ps_fil2_to_1_ps(__m128 in0, __m128 in1, __m128 in2, __m128 in3)
{
__m128 _c01, _c23;
}
/* Sum the elements within each input register and return the sums */
-static gmx_inline __m128
+static gmx_inline __m128 gmx_simdcall
gmx_mm_transpose_sum4_pr(__m256 in0, __m256 in1,
__m256 in2, __m256 in3)
{
}
/* Sum the elements of halfs of each input register and return the sums */
-static gmx_inline __m128
+static gmx_inline __m128 gmx_simdcall
gmx_mm_transpose_sum4h_pr(__m256 in0, __m256 in2)
{
in0 = _mm256_hadd_ps(in0, _mm256_setzero_ps());
}
/* Put two 128-bit 4-float registers into one 256-bit 8-float register */
-static gmx_inline __m256
+static gmx_inline __m256 gmx_simdcall
gmx_2_mm_to_m256(__m128 in0, __m128 in1)
{
return _mm256_insertf128_ps(_mm256_castps128_ps256(in0), in1, 1);
for (p = 0; p < UNROLLJ; p++)
{
/* Here we load 4 aligned floats, but we need just 2 */
- clj_S[p] = _mm_load_ps(nbfp+type[aj+p]*NBFP_STRIDE);
+ clj_S[p] = _mm_load_ps(nbfp+type[aj+p]*nbfp_stride);
}
gmx_shuffle_4_ps_fil01_to_2_ps(clj_S[0], clj_S[1], clj_S[2], clj_S[3],
&c6t_S[0], &c12t_S[0]);
for (p = 0; p < UNROLLJ; p++)
{
/* Here we load 4 aligned floats, but we need just 2 */
- clj_S0[p] = _mm_load_ps(nbfp0+type[aj+p]*NBFP_STRIDE);
+ clj_S0[p] = _mm_load_ps(nbfp0+type[aj+p]*nbfp_stride);
}
for (p = 0; p < UNROLLJ; p++)
{
/* Here we load 4 aligned floats, but we need just 2 */
- clj_S1[p] = _mm_load_ps(nbfp1+type[aj+p]*NBFP_STRIDE);
+ clj_S1[p] = _mm_load_ps(nbfp1+type[aj+p]*nbfp_stride);
}
gmx_shuffle_4_ps_fil01_to_2_ps(clj_S0[0], clj_S0[1], clj_S0[2], clj_S0[3],
&c6t_S[0], &c12t_S[0]);
* This is only faster when we use FDV0 formatted tables, where we also need
* to multiple the index by 4, which can be done by a SIMD bit shift.
* With single precision AVX, 8 extracts are much slower than 1 store.
- * Because of this, the load_table_f macro always takes the ti parameter,
- * but it is only used with AVX.
- */
+ * Because of this, the load_table_f function always takes the ti
+ * parameter, which should contain a buffer that is aligned with
+ * prepare_table_load_buffer(), but it is only used with full-width
+ * AVX_256. */
-static gmx_inline void
-load_table_f(const real *tab_coul_FDV0, gmx_epi32 ti_S, int *ti,
+static gmx_inline void gmx_simdcall
+load_table_f(const real *tab_coul_FDV0, gmx_simd_int32_t ti_S, int *ti,
__m256 *ctab0_S, __m256 *ctab1_S)
{
__m128 ctab_S[8], ctabt_S[4];
*ctab1_S = gmx_2_mm_to_m256(ctabt_S[2], ctabt_S[3]);
}
-static gmx_inline void
-load_table_f_v(const real *tab_coul_FDV0, gmx_epi32 ti_S, int *ti,
+static gmx_inline void gmx_simdcall
+load_table_f_v(const real *tab_coul_FDV0, gmx_simd_int32_t ti_S, int *ti,
__m256 *ctab0_S, __m256 *ctab1_S, __m256 *ctabv_S)
{
__m128 ctab_S[8], ctabt_S[4], ctabvt_S[2];
*ctabv_S = gmx_2_mm_to_m256(ctabvt_S[0], ctabvt_S[1]);
}
+#ifdef GMX_SIMD_HAVE_FINT32_LOGICAL
+
+typedef gmx_simd_int32_t gmx_exclfilter;
+static const int filter_stride = GMX_SIMD_INT32_WIDTH/GMX_SIMD_REAL_WIDTH;
+
+static gmx_inline gmx_exclfilter gmx_simdcall
+gmx_load1_exclfilter(int e)
+{
+ return _mm256_set1_epi32(e);
+}
+
+static gmx_inline gmx_exclfilter gmx_simdcall
+gmx_load_exclusion_filter(const unsigned *i)
+{
+ return gmx_simd_load_i(i);
+}
+
+static gmx_inline gmx_simd_bool_t gmx_simdcall
+gmx_checkbitmask_pb(gmx_exclfilter m0, gmx_exclfilter m1)
+{
+ return _mm256_castsi256_ps(_mm256_cmpeq_epi32(_mm256_andnot_si256(m0, m1), _mm256_setzero_si256()));
+}
+
+#else /* GMX_SIMD_HAVE_FINT32_LOGICAL */
+
+/* No integer support, use a real to store the exclusion bits */
+typedef gmx_simd_real_t gmx_exclfilter;
+static const int filter_stride = 1;
+
+static gmx_inline gmx_exclfilter gmx_simdcall
+gmx_load1_exclfilter(int e)
+{
+ return _mm256_castsi256_ps(_mm256_set1_epi32(e));
+}
+
+static gmx_inline gmx_exclfilter gmx_simdcall
+gmx_load_exclusion_filter(const unsigned *i)
+{
+ return gmx_simd_load_r((real *) (i));
+}
+
+static gmx_inline gmx_simd_bool_t gmx_simdcall
+gmx_checkbitmask_pb(gmx_exclfilter m0, gmx_exclfilter m1)
+{
+ return _mm256_cmp_ps(_mm256_cvtepi32_ps(_mm256_castps_si256(_mm256_and_ps(m0, m1))), _mm256_setzero_ps(), 0x0c);
+}
+
+#endif /* GMX_SIMD_HAVE_FINT32_LOGICAL */
+
#endif /* _nbnxn_kernel_simd_utils_x86_s256s_h_ */
biod.pnpi.spb.ru / alexxy / gromacs.git / blobdiff