/* Without exclusions and energies we only need to mask the cut-off,
* this can be faster with blendv.
*/
-#if !(defined CHECK_EXCLS || defined CALC_ENERGIES) && defined GMX_HAVE_SIMD_BLENDV && !defined COUNT_PAIRS
+#if !(defined CHECK_EXCLS || defined CALC_ENERGIES) && defined GMX_SIMD_HAVE_BLENDV && !defined COUNT_PAIRS
/* With RF and tabulated Coulomb we replace cmp+and with sub+blendv.
* With gcc this is slower, except for RF on Sandy Bridge.
* Tested with gcc 4.6.2, 4.6.3 and 4.7.1.
#ifdef CHECK_EXCLS
/* Interaction (non-exclusion) mask of all 1's or 0's */
- gmx_mm_pr int_S0;
- gmx_mm_pr int_S2;
+ gmx_mm_pb interact_S0;
+ gmx_mm_pb interact_S2;
#endif
gmx_mm_pr jx_S, jy_S, jz_S;
gmx_mm_pr rsq_S2, rinv_S2, rinvsq_S2;
#ifndef CUTOFF_BLENDV
/* wco: within cut-off, mask of all 1's or 0's */
- gmx_mm_pr wco_S0;
- gmx_mm_pr wco_S2;
+ gmx_mm_pb wco_S0;
+ gmx_mm_pb wco_S2;
#endif
#ifdef VDW_CUTOFF_CHECK
- gmx_mm_pr wco_vdw_S0;
+ gmx_mm_pb wco_vdw_S0;
#ifndef HALF_LJ
- gmx_mm_pr wco_vdw_S2;
+ gmx_mm_pb wco_vdw_S2;
#endif
#endif
#ifdef CALC_COULOMB
ajz = ajy + STRIDE;
#ifdef CHECK_EXCLS
+#ifdef GMX_SIMD_HAVE_CHECKBITMASK_EPI32
{
- /* Load integer interaction mask */
+ /* Load integer topology exclusion interaction mask */
+ gmx_epi32 mask_pr_S = gmx_set1_epi32(l_cj[cjind].excl);
+
+ interact_S0 = gmx_checkbitmask_epi32(mask_pr_S, filter_S0);
+ interact_S2 = gmx_checkbitmask_epi32(mask_pr_S, filter_S2);
+ }
+#else
+#ifdef GMX_SIMD_HAVE_CHECKBITMASK_PR
+ {
+ /* Integer mask set, cast to real and real mask operations */
gmx_mm_pr mask_pr_S = gmx_castsi_pr(gmx_set1_epi32(l_cj[cjind].excl));
- int_S0 = gmx_checkbitmask_pr(mask_pr_S, mask_S0);
- int_S2 = gmx_checkbitmask_pr(mask_pr_S, mask_S2);
+ interact_S0 = gmx_checkbitmask_pr(mask_pr_S, filter_S0);
+ interact_S2 = gmx_checkbitmask_pr(mask_pr_S, filter_S2);
}
+#else
+#error "No SIMD bitmask operation available"
+#endif
#endif
+#endif /* CHECK_EXCLS */
/* load j atom coordinates */
- gmx_loaddh_pr(jx_S, x+ajx);
- gmx_loaddh_pr(jy_S, x+ajy);
- gmx_loaddh_pr(jz_S, x+ajz);
+ gmx_loaddh_pr(&jx_S, x+ajx);
+ gmx_loaddh_pr(&jy_S, x+ajy);
+ gmx_loaddh_pr(&jz_S, x+ajz);
/* Calculate distance */
dx_S0 = gmx_sub_pr(ix_S0, jx_S);
#if UNROLLJ == UNROLLI
if (cj == ci_sh)
{
- wco_S0 = gmx_and_pr(wco_S0, diag_S0);
- wco_S2 = gmx_and_pr(wco_S2, diag_S2);
+ wco_S0 = gmx_and_pb(wco_S0, diagonal_mask_S0);
+ wco_S2 = gmx_and_pb(wco_S2, diagonal_mask_S2);
}
#else
#if UNROLLJ == 2*UNROLLI
if (cj*2 == ci_sh)
{
- wco_S0 = gmx_and_pr(wco_S0, diag0_S0);
- wco_S2 = gmx_and_pr(wco_S2, diag0_S2);
+ wco_S0 = gmx_and_pb(wco_S0, diagonal_mask0_S0);
+ wco_S2 = gmx_and_pb(wco_S2, diagonal_mask0_S2);
}
else if (cj*2 + 1 == ci_sh)
{
- wco_S0 = gmx_and_pr(wco_S0, diag1_S0);
- wco_S2 = gmx_and_pr(wco_S2, diag1_S2);
+ wco_S0 = gmx_and_pb(wco_S0, diagonal_mask1_S0);
+ wco_S2 = gmx_and_pb(wco_S2, diagonal_mask1_S2);
}
#else
#error "only UNROLLJ == UNROLLI*(1 or 2) currently supported in 2xnn kernels"
#endif
#else /* EXCL_FORCES */
/* No exclusion forces: remove all excluded atom pairs from the list */
- wco_S0 = gmx_and_pr(wco_S0, int_S0);
- wco_S2 = gmx_and_pr(wco_S2, int_S2);
+ wco_S0 = gmx_and_pb(wco_S0, interact_S0);
+ wco_S2 = gmx_and_pb(wco_S2, interact_S2);
#endif
#endif
#ifdef CHECK_EXCLS
/* For excluded pairs add a small number to avoid r^-6 = NaN */
- rsq_S0 = gmx_add_pr(rsq_S0, gmx_andnot_pr(int_S0, avoid_sing_S));
- rsq_S2 = gmx_add_pr(rsq_S2, gmx_andnot_pr(int_S2, avoid_sing_S));
+ rsq_S0 = gmx_masknot_add_pr(interact_S0, rsq_S0, avoid_sing_S);
+ rsq_S2 = gmx_masknot_add_pr(interact_S2, rsq_S2, avoid_sing_S);
#endif
/* Calculate 1/r */
#ifdef CALC_COULOMB
/* Load parameters for j atom */
- gmx_loaddh_pr(jq_S, q+aj);
+ gmx_loaddh_pr(&jq_S, q+aj);
qq_S0 = gmx_mul_pr(iq_S0, jq_S);
qq_S2 = gmx_mul_pr(iq_S2, jq_S);
#endif
#ifdef CALC_LJ
#if !defined LJ_COMB_GEOM && !defined LJ_COMB_LB && !defined FIX_LJ_C
- load_lj_pair_params2(nbfp0, nbfp1, type, aj, c6_S0, c12_S0);
+ load_lj_pair_params2(nbfp0, nbfp1, type, aj, &c6_S0, &c12_S0);
#ifndef HALF_LJ
- load_lj_pair_params2(nbfp2, nbfp3, type, aj, c6_S2, c12_S2);
+ load_lj_pair_params2(nbfp2, nbfp3, type, aj, &c6_S2, &c12_S2);
#endif
#endif /* not defined any LJ rule */
#ifdef LJ_COMB_GEOM
- gmx_loaddh_pr(c6s_j_S, ljc+aj2+0);
- gmx_loaddh_pr(c12s_j_S, ljc+aj2+STRIDE);
+ gmx_loaddh_pr(&c6s_j_S, ljc+aj2+0);
+ gmx_loaddh_pr(&c12s_j_S, ljc+aj2+STRIDE);
c6_S0 = gmx_mul_pr(c6s_S0, c6s_j_S );
#ifndef HALF_LJ
c6_S2 = gmx_mul_pr(c6s_S2, c6s_j_S );
#endif /* LJ_COMB_GEOM */
#ifdef LJ_COMB_LB
- gmx_loaddh_pr(hsig_j_S, ljc+aj2+0);
- gmx_loaddh_pr(seps_j_S, ljc+aj2+STRIDE);
+ gmx_loaddh_pr(&hsig_j_S, ljc+aj2+0);
+ gmx_loaddh_pr(&seps_j_S, ljc+aj2+STRIDE);
sig_S0 = gmx_add_pr(hsig_i_S0, hsig_j_S);
eps_S0 = gmx_mul_pr(seps_i_S0, seps_j_S);
#ifdef EXCL_FORCES
/* Only add 1/r for non-excluded atom pairs */
- rinv_ex_S0 = gmx_blendzero_pr(rinv_S0, int_S0);
- rinv_ex_S2 = gmx_blendzero_pr(rinv_S2, int_S2);
+ rinv_ex_S0 = gmx_blendzero_pr(rinv_S0, interact_S0);
+ rinv_ex_S2 = gmx_blendzero_pr(rinv_S2, interact_S2);
#else
/* No exclusion forces, we always need 1/r */
#define rinv_ex_S0 rinv_S0
#ifdef CALC_COUL_RF
/* Electrostatic interactions */
- frcoul_S0 = gmx_mul_pr(qq_S0, gmx_add_pr(rinv_ex_S0, gmx_mul_pr(rsq_S0, mrc_3_S)));
- frcoul_S2 = gmx_mul_pr(qq_S2, gmx_add_pr(rinv_ex_S2, gmx_mul_pr(rsq_S2, mrc_3_S)));
+ frcoul_S0 = gmx_mul_pr(qq_S0, gmx_madd_pr(rsq_S0, mrc_3_S, rinv_ex_S0));
+ frcoul_S2 = gmx_mul_pr(qq_S2, gmx_madd_pr(rsq_S2, mrc_3_S, rinv_ex_S2));
#ifdef CALC_ENERGIES
vcoul_S0 = gmx_mul_pr(qq_S0, gmx_add_pr(rinv_ex_S0, gmx_add_pr(gmx_mul_pr(rsq_S0, hrc_3_S), moh_rc_S)));
#endif
ewcorr_S0 = gmx_mul_pr(gmx_pmecorrF_pr(brsq_S0), beta_S);
ewcorr_S2 = gmx_mul_pr(gmx_pmecorrF_pr(brsq_S2), beta_S);
- frcoul_S0 = gmx_mul_pr(qq_S0, gmx_add_pr(rinv_ex_S0, gmx_mul_pr(ewcorr_S0, brsq_S0)));
- frcoul_S2 = gmx_mul_pr(qq_S2, gmx_add_pr(rinv_ex_S2, gmx_mul_pr(ewcorr_S2, brsq_S2)));
+ frcoul_S0 = gmx_mul_pr(qq_S0, gmx_madd_pr(ewcorr_S0, brsq_S0, rinv_ex_S0));
+ frcoul_S2 = gmx_mul_pr(qq_S2, gmx_madd_pr(ewcorr_S2, brsq_S2, rinv_ex_S2));
#ifdef CALC_ENERGIES
vc_sub_S0 = gmx_mul_pr(gmx_pmecorrV_pr(brsq_S0), beta_S);
/* Truncate scaled r to an int */
ti_S0 = gmx_cvttpr_epi32(rs_S0);
ti_S2 = gmx_cvttpr_epi32(rs_S2);
-#ifdef GMX_HAVE_SIMD_FLOOR
+#ifdef GMX_SIMD_HAVE_FLOOR
rf_S0 = gmx_floor_pr(rs_S0);
rf_S2 = gmx_floor_pr(rs_S2);
#else
* Currently single precision uses FDV0, double F and V.
*/
#ifndef CALC_ENERGIES
- load_table_f(tab_coul_F, ti_S0, ti0, ctab0_S0, ctab1_S0);
- load_table_f(tab_coul_F, ti_S2, ti2, ctab0_S2, ctab1_S2);
+ load_table_f(tab_coul_F, ti_S0, ti0, &ctab0_S0, &ctab1_S0);
+ load_table_f(tab_coul_F, ti_S2, ti2, &ctab0_S2, &ctab1_S2);
#else
#ifdef TAB_FDV0
- load_table_f_v(tab_coul_F, ti_S0, ti0, ctab0_S0, ctab1_S0, ctabv_S0);
- load_table_f_v(tab_coul_F, ti_S2, ti2, ctab0_S2, ctab1_S2, ctabv_S2);
+ load_table_f_v(tab_coul_F, ti_S0, ti0, &ctab0_S0, &ctab1_S0, &ctabv_S0);
+ load_table_f_v(tab_coul_F, ti_S2, ti2, &ctab0_S2, &ctab1_S2, &ctabv_S2);
#else
- load_table_f_v(tab_coul_F, tab_coul_V, ti_S0, ti0, ctab0_S0, ctab1_S0, ctabv_S0);
- load_table_f_v(tab_coul_F, tab_coul_V, ti_S2, ti2, ctab0_S2, ctab1_S2, ctabv_S2);
+ load_table_f_v(tab_coul_F, tab_coul_V, ti_S0, ti0, &ctab0_S0, &ctab1_S0, &ctabv_S0);
+ load_table_f_v(tab_coul_F, tab_coul_V, ti_S2, ti2, &ctab0_S2, &ctab1_S2, &ctabv_S2);
#endif
#endif
fsub_S0 = gmx_add_pr(ctab0_S0, gmx_mul_pr(frac_S0, ctab1_S0));
#ifndef NO_SHIFT_EWALD
/* Add Ewald potential shift to vc_sub for convenience */
#ifdef CHECK_EXCLS
- vc_sub_S0 = gmx_add_pr(vc_sub_S0, gmx_blendzero_pr(sh_ewald_S, int_S0));
- vc_sub_S2 = gmx_add_pr(vc_sub_S2, gmx_blendzero_pr(sh_ewald_S, int_S2));
+ vc_sub_S0 = gmx_add_pr(vc_sub_S0, gmx_blendzero_pr(sh_ewald_S, interact_S0));
+ vc_sub_S2 = gmx_add_pr(vc_sub_S2, gmx_blendzero_pr(sh_ewald_S, interact_S2));
#else
vc_sub_S0 = gmx_add_pr(vc_sub_S0, sh_ewald_S);
vc_sub_S2 = gmx_add_pr(vc_sub_S2, sh_ewald_S);
#ifndef LJ_COMB_LB
rinvsix_S0 = gmx_mul_pr(rinvsq_S0, gmx_mul_pr(rinvsq_S0, rinvsq_S0));
#ifdef EXCL_FORCES
- rinvsix_S0 = gmx_blendzero_pr(rinvsix_S0, int_S0);
+ rinvsix_S0 = gmx_blendzero_pr(rinvsix_S0, interact_S0);
#endif
#ifndef HALF_LJ
rinvsix_S2 = gmx_mul_pr(rinvsq_S2, gmx_mul_pr(rinvsq_S2, rinvsq_S2));
#ifdef EXCL_FORCES
- rinvsix_S2 = gmx_blendzero_pr(rinvsix_S2, int_S2);
+ rinvsix_S2 = gmx_blendzero_pr(rinvsix_S2, interact_S2);
#endif
#endif
#ifdef VDW_CUTOFF_CHECK
#endif
sir6_S0 = gmx_mul_pr(sir2_S0, gmx_mul_pr(sir2_S0, sir2_S0));
#ifdef EXCL_FORCES
- sir6_S0 = gmx_blendzero_pr(sir6_S0, int_S0);
+ sir6_S0 = gmx_blendzero_pr(sir6_S0, interact_S0);
#endif
#ifndef HALF_LJ
sir6_S2 = gmx_mul_pr(sir2_S2, gmx_mul_pr(sir2_S2, sir2_S2));
#ifdef EXCL_FORCES
- sir6_S2 = gmx_blendzero_pr(sir6_S2, int_S2);
+ sir6_S2 = gmx_blendzero_pr(sir6_S2, interact_S2);
#endif
#endif
#ifdef VDW_CUTOFF_CHECK
#endif
#ifdef CHECK_EXCLS
/* The potential shift should be removed for excluded pairs */
- VLJ_S0 = gmx_blendzero_pr(VLJ_S0, int_S0);
+ VLJ_S0 = gmx_blendzero_pr(VLJ_S0, interact_S0);
#ifndef HALF_LJ
- VLJ_S2 = gmx_blendzero_pr(VLJ_S2, int_S2);
+ VLJ_S2 = gmx_blendzero_pr(VLJ_S2, interact_S2);
#endif
#endif
#ifndef ENERGY_GROUPS
fiz_S2 = gmx_add_pr(fiz_S2, tz_S2);
/* Decrement j atom force */
- gmx_load_hpr(fjx_S, f+ajx);
- gmx_load_hpr(fjy_S, f+ajy);
- gmx_load_hpr(fjz_S, f+ajz);
+ gmx_load_hpr(&fjx_S, f+ajx);
+ gmx_load_hpr(&fjy_S, f+ajy);
+ gmx_load_hpr(&fjz_S, f+ajz);
gmx_store_hpr(f+ajx, gmx_sub_hpr(fjx_S, gmx_sum4_hpr(tx_S0, tx_S2)));
gmx_store_hpr(f+ajy, gmx_sub_hpr(fjy_S, gmx_sum4_hpr(ty_S0, ty_S2)));
gmx_store_hpr(f+ajz, gmx_sub_hpr(fjz_S, gmx_sum4_hpr(tz_S0, tz_S2)));
biod.pnpi.spb.ru / alexxy / gromacs.git / blobdiff