src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_avx_128_fma_single.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
  37  */
  38 #include "config.h"
  39
  40 #include <math.h>
  41
  42 #include "../nb_kernel.h"
  43 #include "types/simple.h"
  44 #include "gromacs/math/vec.h"
  45 #include "nrnb.h"
  46
  47 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
  48 #include "kernelutil_x86_avx_128_fma_single.h"
  49
  50 /*
  51  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  52  * Electrostatics interaction: CubicSplineTable
  53  * VdW interaction:            LennardJones
  54  * Geometry:                   Water3-Water3
  55  * Calculate force/pot:        PotentialAndForce
  56  */
  57 void
  58 nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  59                     (t_nblist                    * gmx_restrict       nlist,
  60                      rvec                        * gmx_restrict          xx,
  61                      rvec                        * gmx_restrict          ff,
  62                      t_forcerec                  * gmx_restrict          fr,
  63                      t_mdatoms                   * gmx_restrict     mdatoms,
  64                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  65                      t_nrnb                      * gmx_restrict        nrnb)
  66 {
  67     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  68      * just 0 for non-waters.
  69      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  70      * jnr indices corresponding to data put in the four positions in the SIMD register.
  71      */
  72     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  73     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  74     int              jnrA,jnrB,jnrC,jnrD;
  75     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  76     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  77     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  78     real             rcutoff_scalar;
  79     real             *shiftvec,*fshift,*x,*f;
  80     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  81     real             scratch[4*DIM];
  82     __m128           fscal,rcutoff,rcutoff2,jidxall;
  83     int              vdwioffset0;
  84     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  85     int              vdwioffset1;
  86     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  87     int              vdwioffset2;
  88     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  89     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  90     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  91     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
  92     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  93     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
  94     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  95     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  96     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
  97     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
  98     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  99     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 100     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 101     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 102     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 103     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 104     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 105     real             *charge;
 106     int              nvdwtype;
 107     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 108     int              *vdwtype;
 109     real             *vdwparam;
 110     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 111     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 112     __m128i          vfitab;
 113     __m128i          ifour       = _mm_set1_epi32(4);
 114     __m128           rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
 115     real             *vftab;
 116     __m128           dummy_mask,cutoff_mask;
 117     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 118     __m128           one     = _mm_set1_ps(1.0);
 119     __m128           two     = _mm_set1_ps(2.0);
 120     x                = xx[0];
 121     f                = ff[0];
 122
 123     nri              = nlist->nri;
 124     iinr             = nlist->iinr;
 125     jindex           = nlist->jindex;
 126     jjnr             = nlist->jjnr;
 127     shiftidx         = nlist->shift;
 128     gid              = nlist->gid;
 129     shiftvec         = fr->shift_vec[0];
 130     fshift           = fr->fshift[0];
 131     facel            = _mm_set1_ps(fr->epsfac);
 132     charge           = mdatoms->chargeA;
 133     nvdwtype         = fr->ntype;
 134     vdwparam         = fr->nbfp;
 135     vdwtype          = mdatoms->typeA;
 136
 137     vftab            = kernel_data->table_elec->data;
 138     vftabscale       = _mm_set1_ps(kernel_data->table_elec->scale);
 139
 140     /* Setup water-specific parameters */
 141     inr              = nlist->iinr[0];
 142     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 143     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 144     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 145     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 146
 147     jq0              = _mm_set1_ps(charge[inr+0]);
 148     jq1              = _mm_set1_ps(charge[inr+1]);
 149     jq2              = _mm_set1_ps(charge[inr+2]);
 150     vdwjidx0A        = 2*vdwtype[inr+0];
 151     qq00             = _mm_mul_ps(iq0,jq0);
 152     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 153     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 154     qq01             = _mm_mul_ps(iq0,jq1);
 155     qq02             = _mm_mul_ps(iq0,jq2);
 156     qq10             = _mm_mul_ps(iq1,jq0);
 157     qq11             = _mm_mul_ps(iq1,jq1);
 158     qq12             = _mm_mul_ps(iq1,jq2);
 159     qq20             = _mm_mul_ps(iq2,jq0);
 160     qq21             = _mm_mul_ps(iq2,jq1);
 161     qq22             = _mm_mul_ps(iq2,jq2);
 162
 163     /* Avoid stupid compiler warnings */
 164     jnrA = jnrB = jnrC = jnrD = 0;
 165     j_coord_offsetA = 0;
 166     j_coord_offsetB = 0;
 167     j_coord_offsetC = 0;
 168     j_coord_offsetD = 0;
 169
 170     outeriter        = 0;
 171     inneriter        = 0;
 172
 173     for(iidx=0;iidx<4*DIM;iidx++)
 174     {
 175         scratch[iidx] = 0.0;
 176     }
 177
 178     /* Start outer loop over neighborlists */
 179     for(iidx=0; iidx<nri; iidx++)
 180     {
 181         /* Load shift vector for this list */
 182         i_shift_offset   = DIM*shiftidx[iidx];
 183
 184         /* Load limits for loop over neighbors */
 185         j_index_start    = jindex[iidx];
 186         j_index_end      = jindex[iidx+1];
 187
 188         /* Get outer coordinate index */
 189         inr              = iinr[iidx];
 190         i_coord_offset   = DIM*inr;
 191
 192         /* Load i particle coords and add shift vector */
 193         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 194                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 195
 196         fix0             = _mm_setzero_ps();
 197         fiy0             = _mm_setzero_ps();
 198         fiz0             = _mm_setzero_ps();
 199         fix1             = _mm_setzero_ps();
 200         fiy1             = _mm_setzero_ps();
 201         fiz1             = _mm_setzero_ps();
 202         fix2             = _mm_setzero_ps();
 203         fiy2             = _mm_setzero_ps();
 204         fiz2             = _mm_setzero_ps();
 205
 206         /* Reset potential sums */
 207         velecsum         = _mm_setzero_ps();
 208         vvdwsum          = _mm_setzero_ps();
 209
 210         /* Start inner kernel loop */
 211         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 212         {
 213
 214             /* Get j neighbor index, and coordinate index */
 215             jnrA             = jjnr[jidx];
 216             jnrB             = jjnr[jidx+1];
 217             jnrC             = jjnr[jidx+2];
 218             jnrD             = jjnr[jidx+3];
 219             j_coord_offsetA  = DIM*jnrA;
 220             j_coord_offsetB  = DIM*jnrB;
 221             j_coord_offsetC  = DIM*jnrC;
 222             j_coord_offsetD  = DIM*jnrD;
 223
 224             /* load j atom coordinates */
 225             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 226                                               x+j_coord_offsetC,x+j_coord_offsetD,
 227                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 228
 229             /* Calculate displacement vector */
 230             dx00             = _mm_sub_ps(ix0,jx0);
 231             dy00             = _mm_sub_ps(iy0,jy0);
 232             dz00             = _mm_sub_ps(iz0,jz0);
 233             dx01             = _mm_sub_ps(ix0,jx1);
 234             dy01             = _mm_sub_ps(iy0,jy1);
 235             dz01             = _mm_sub_ps(iz0,jz1);
 236             dx02             = _mm_sub_ps(ix0,jx2);
 237             dy02             = _mm_sub_ps(iy0,jy2);
 238             dz02             = _mm_sub_ps(iz0,jz2);
 239             dx10             = _mm_sub_ps(ix1,jx0);
 240             dy10             = _mm_sub_ps(iy1,jy0);
 241             dz10             = _mm_sub_ps(iz1,jz0);
 242             dx11             = _mm_sub_ps(ix1,jx1);
 243             dy11             = _mm_sub_ps(iy1,jy1);
 244             dz11             = _mm_sub_ps(iz1,jz1);
 245             dx12             = _mm_sub_ps(ix1,jx2);
 246             dy12             = _mm_sub_ps(iy1,jy2);
 247             dz12             = _mm_sub_ps(iz1,jz2);
 248             dx20             = _mm_sub_ps(ix2,jx0);
 249             dy20             = _mm_sub_ps(iy2,jy0);
 250             dz20             = _mm_sub_ps(iz2,jz0);
 251             dx21             = _mm_sub_ps(ix2,jx1);
 252             dy21             = _mm_sub_ps(iy2,jy1);
 253             dz21             = _mm_sub_ps(iz2,jz1);
 254             dx22             = _mm_sub_ps(ix2,jx2);
 255             dy22             = _mm_sub_ps(iy2,jy2);
 256             dz22             = _mm_sub_ps(iz2,jz2);
 257
 258             /* Calculate squared distance and things based on it */
 259             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 260             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 261             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 262             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 263             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 264             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 265             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 266             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 267             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 268
 269             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 270             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 271             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 272             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 273             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 274             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 275             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 276             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 277             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 278
 279             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 280
 281             fjx0             = _mm_setzero_ps();
 282             fjy0             = _mm_setzero_ps();
 283             fjz0             = _mm_setzero_ps();
 284             fjx1             = _mm_setzero_ps();
 285             fjy1             = _mm_setzero_ps();
 286             fjz1             = _mm_setzero_ps();
 287             fjx2             = _mm_setzero_ps();
 288             fjy2             = _mm_setzero_ps();
 289             fjz2             = _mm_setzero_ps();
 290
 291             /**************************
 292              * CALCULATE INTERACTIONS *
 293              **************************/
 294
 295             r00              = _mm_mul_ps(rsq00,rinv00);
 296
 297             /* Calculate table index by multiplying r with table scale and truncate to integer */
 298             rt               = _mm_mul_ps(r00,vftabscale);
 299             vfitab           = _mm_cvttps_epi32(rt);
 300 #ifdef __XOP__
 301             vfeps            = _mm_frcz_ps(rt);
 302 #else
 303             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 304 #endif
 305             twovfeps         = _mm_add_ps(vfeps,vfeps);
 306             vfitab           = _mm_slli_epi32(vfitab,2);
 307
 308             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 309             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 310             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 311             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 312             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 313             _MM_TRANSPOSE4_PS(Y,F,G,H);
 314             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 315             VV               = _mm_macc_ps(vfeps,Fp,Y);
 316             velec            = _mm_mul_ps(qq00,VV);
 317             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 318             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
 319
 320             /* LENNARD-JONES DISPERSION/REPULSION */
 321
 322             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 323             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 324             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 325             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 326             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 327
 328             /* Update potential sum for this i atom from the interaction with this j atom. */
 329             velecsum         = _mm_add_ps(velecsum,velec);
 330             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 331
 332             fscal            = _mm_add_ps(felec,fvdw);
 333
 334              /* Update vectorial force */
 335             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 336             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 337             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 338
 339             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 340             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 341             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 342
 343             /**************************
 344              * CALCULATE INTERACTIONS *
 345              **************************/
 346
 347             r01              = _mm_mul_ps(rsq01,rinv01);
 348
 349             /* Calculate table index by multiplying r with table scale and truncate to integer */
 350             rt               = _mm_mul_ps(r01,vftabscale);
 351             vfitab           = _mm_cvttps_epi32(rt);
 352 #ifdef __XOP__
 353             vfeps            = _mm_frcz_ps(rt);
 354 #else
 355             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 356 #endif
 357             twovfeps         = _mm_add_ps(vfeps,vfeps);
 358             vfitab           = _mm_slli_epi32(vfitab,2);
 359
 360             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 361             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 362             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 363             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 364             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 365             _MM_TRANSPOSE4_PS(Y,F,G,H);
 366             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 367             VV               = _mm_macc_ps(vfeps,Fp,Y);
 368             velec            = _mm_mul_ps(qq01,VV);
 369             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 370             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
 371
 372             /* Update potential sum for this i atom from the interaction with this j atom. */
 373             velecsum         = _mm_add_ps(velecsum,velec);
 374
 375             fscal            = felec;
 376
 377              /* Update vectorial force */
 378             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 379             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 380             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 381
 382             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 383             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 384             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 385
 386             /**************************
 387              * CALCULATE INTERACTIONS *
 388              **************************/
 389
 390             r02              = _mm_mul_ps(rsq02,rinv02);
 391
 392             /* Calculate table index by multiplying r with table scale and truncate to integer */
 393             rt               = _mm_mul_ps(r02,vftabscale);
 394             vfitab           = _mm_cvttps_epi32(rt);
 395 #ifdef __XOP__
 396             vfeps            = _mm_frcz_ps(rt);
 397 #else
 398             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 399 #endif
 400             twovfeps         = _mm_add_ps(vfeps,vfeps);
 401             vfitab           = _mm_slli_epi32(vfitab,2);
 402
 403             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 404             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 405             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 406             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 407             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 408             _MM_TRANSPOSE4_PS(Y,F,G,H);
 409             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 410             VV               = _mm_macc_ps(vfeps,Fp,Y);
 411             velec            = _mm_mul_ps(qq02,VV);
 412             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 413             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
 414
 415             /* Update potential sum for this i atom from the interaction with this j atom. */
 416             velecsum         = _mm_add_ps(velecsum,velec);
 417
 418             fscal            = felec;
 419
 420              /* Update vectorial force */
 421             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 422             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 423             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 424
 425             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 426             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 427             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 428
 429             /**************************
 430              * CALCULATE INTERACTIONS *
 431              **************************/
 432
 433             r10              = _mm_mul_ps(rsq10,rinv10);
 434
 435             /* Calculate table index by multiplying r with table scale and truncate to integer */
 436             rt               = _mm_mul_ps(r10,vftabscale);
 437             vfitab           = _mm_cvttps_epi32(rt);
 438 #ifdef __XOP__
 439             vfeps            = _mm_frcz_ps(rt);
 440 #else
 441             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 442 #endif
 443             twovfeps         = _mm_add_ps(vfeps,vfeps);
 444             vfitab           = _mm_slli_epi32(vfitab,2);
 445
 446             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 447             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 448             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 449             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 450             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 451             _MM_TRANSPOSE4_PS(Y,F,G,H);
 452             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 453             VV               = _mm_macc_ps(vfeps,Fp,Y);
 454             velec            = _mm_mul_ps(qq10,VV);
 455             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 456             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
 457
 458             /* Update potential sum for this i atom from the interaction with this j atom. */
 459             velecsum         = _mm_add_ps(velecsum,velec);
 460
 461             fscal            = felec;
 462
 463              /* Update vectorial force */
 464             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 465             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 466             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 467
 468             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 469             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 470             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 471
 472             /**************************
 473              * CALCULATE INTERACTIONS *
 474              **************************/
 475
 476             r11              = _mm_mul_ps(rsq11,rinv11);
 477
 478             /* Calculate table index by multiplying r with table scale and truncate to integer */
 479             rt               = _mm_mul_ps(r11,vftabscale);
 480             vfitab           = _mm_cvttps_epi32(rt);
 481 #ifdef __XOP__
 482             vfeps            = _mm_frcz_ps(rt);
 483 #else
 484             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 485 #endif
 486             twovfeps         = _mm_add_ps(vfeps,vfeps);
 487             vfitab           = _mm_slli_epi32(vfitab,2);
 488
 489             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 490             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 491             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 492             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 493             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 494             _MM_TRANSPOSE4_PS(Y,F,G,H);
 495             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 496             VV               = _mm_macc_ps(vfeps,Fp,Y);
 497             velec            = _mm_mul_ps(qq11,VV);
 498             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 499             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
 500
 501             /* Update potential sum for this i atom from the interaction with this j atom. */
 502             velecsum         = _mm_add_ps(velecsum,velec);
 503
 504             fscal            = felec;
 505
 506              /* Update vectorial force */
 507             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 508             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 509             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 510
 511             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 512             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 513             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 514
 515             /**************************
 516              * CALCULATE INTERACTIONS *
 517              **************************/
 518
 519             r12              = _mm_mul_ps(rsq12,rinv12);
 520
 521             /* Calculate table index by multiplying r with table scale and truncate to integer */
 522             rt               = _mm_mul_ps(r12,vftabscale);
 523             vfitab           = _mm_cvttps_epi32(rt);
 524 #ifdef __XOP__
 525             vfeps            = _mm_frcz_ps(rt);
 526 #else
 527             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 528 #endif
 529             twovfeps         = _mm_add_ps(vfeps,vfeps);
 530             vfitab           = _mm_slli_epi32(vfitab,2);
 531
 532             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 533             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 534             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 535             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 536             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 537             _MM_TRANSPOSE4_PS(Y,F,G,H);
 538             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 539             VV               = _mm_macc_ps(vfeps,Fp,Y);
 540             velec            = _mm_mul_ps(qq12,VV);
 541             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 542             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
 543
 544             /* Update potential sum for this i atom from the interaction with this j atom. */
 545             velecsum         = _mm_add_ps(velecsum,velec);
 546
 547             fscal            = felec;
 548
 549              /* Update vectorial force */
 550             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 551             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 552             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 553
 554             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 555             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 556             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 557
 558             /**************************
 559              * CALCULATE INTERACTIONS *
 560              **************************/
 561
 562             r20              = _mm_mul_ps(rsq20,rinv20);
 563
 564             /* Calculate table index by multiplying r with table scale and truncate to integer */
 565             rt               = _mm_mul_ps(r20,vftabscale);
 566             vfitab           = _mm_cvttps_epi32(rt);
 567 #ifdef __XOP__
 568             vfeps            = _mm_frcz_ps(rt);
 569 #else
 570             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 571 #endif
 572             twovfeps         = _mm_add_ps(vfeps,vfeps);
 573             vfitab           = _mm_slli_epi32(vfitab,2);
 574
 575             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 576             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 577             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 578             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 579             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 580             _MM_TRANSPOSE4_PS(Y,F,G,H);
 581             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 582             VV               = _mm_macc_ps(vfeps,Fp,Y);
 583             velec            = _mm_mul_ps(qq20,VV);
 584             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 585             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
 586
 587             /* Update potential sum for this i atom from the interaction with this j atom. */
 588             velecsum         = _mm_add_ps(velecsum,velec);
 589
 590             fscal            = felec;
 591
 592              /* Update vectorial force */
 593             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 594             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 595             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 596
 597             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 598             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 599             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 600
 601             /**************************
 602              * CALCULATE INTERACTIONS *
 603              **************************/
 604
 605             r21              = _mm_mul_ps(rsq21,rinv21);
 606
 607             /* Calculate table index by multiplying r with table scale and truncate to integer */
 608             rt               = _mm_mul_ps(r21,vftabscale);
 609             vfitab           = _mm_cvttps_epi32(rt);
 610 #ifdef __XOP__
 611             vfeps            = _mm_frcz_ps(rt);
 612 #else
 613             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 614 #endif
 615             twovfeps         = _mm_add_ps(vfeps,vfeps);
 616             vfitab           = _mm_slli_epi32(vfitab,2);
 617
 618             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 619             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 620             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 621             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 622             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 623             _MM_TRANSPOSE4_PS(Y,F,G,H);
 624             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 625             VV               = _mm_macc_ps(vfeps,Fp,Y);
 626             velec            = _mm_mul_ps(qq21,VV);
 627             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 628             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
 629
 630             /* Update potential sum for this i atom from the interaction with this j atom. */
 631             velecsum         = _mm_add_ps(velecsum,velec);
 632
 633             fscal            = felec;
 634
 635              /* Update vectorial force */
 636             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 637             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 638             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 639
 640             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 641             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 642             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 643
 644             /**************************
 645              * CALCULATE INTERACTIONS *
 646              **************************/
 647
 648             r22              = _mm_mul_ps(rsq22,rinv22);
 649
 650             /* Calculate table index by multiplying r with table scale and truncate to integer */
 651             rt               = _mm_mul_ps(r22,vftabscale);
 652             vfitab           = _mm_cvttps_epi32(rt);
 653 #ifdef __XOP__
 654             vfeps            = _mm_frcz_ps(rt);
 655 #else
 656             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 657 #endif
 658             twovfeps         = _mm_add_ps(vfeps,vfeps);
 659             vfitab           = _mm_slli_epi32(vfitab,2);
 660
 661             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 662             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 663             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 664             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 665             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 666             _MM_TRANSPOSE4_PS(Y,F,G,H);
 667             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 668             VV               = _mm_macc_ps(vfeps,Fp,Y);
 669             velec            = _mm_mul_ps(qq22,VV);
 670             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 671             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
 672
 673             /* Update potential sum for this i atom from the interaction with this j atom. */
 674             velecsum         = _mm_add_ps(velecsum,velec);
 675
 676             fscal            = felec;
 677
 678              /* Update vectorial force */
 679             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 680             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 681             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 682
 683             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 684             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 685             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 686
 687             fjptrA             = f+j_coord_offsetA;
 688             fjptrB             = f+j_coord_offsetB;
 689             fjptrC             = f+j_coord_offsetC;
 690             fjptrD             = f+j_coord_offsetD;
 691
 692             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 693                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 694
 695             /* Inner loop uses 427 flops */
 696         }
 697
 698         if(jidx<j_index_end)
 699         {
 700
 701             /* Get j neighbor index, and coordinate index */
 702             jnrlistA         = jjnr[jidx];
 703             jnrlistB         = jjnr[jidx+1];
 704             jnrlistC         = jjnr[jidx+2];
 705             jnrlistD         = jjnr[jidx+3];
 706             /* Sign of each element will be negative for non-real atoms.
 707              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 708              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 709              */
 710             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 711             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 712             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 713             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 714             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 715             j_coord_offsetA  = DIM*jnrA;
 716             j_coord_offsetB  = DIM*jnrB;
 717             j_coord_offsetC  = DIM*jnrC;
 718             j_coord_offsetD  = DIM*jnrD;
 719
 720             /* load j atom coordinates */
 721             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 722                                               x+j_coord_offsetC,x+j_coord_offsetD,
 723                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 724
 725             /* Calculate displacement vector */
 726             dx00             = _mm_sub_ps(ix0,jx0);
 727             dy00             = _mm_sub_ps(iy0,jy0);
 728             dz00             = _mm_sub_ps(iz0,jz0);
 729             dx01             = _mm_sub_ps(ix0,jx1);
 730             dy01             = _mm_sub_ps(iy0,jy1);
 731             dz01             = _mm_sub_ps(iz0,jz1);
 732             dx02             = _mm_sub_ps(ix0,jx2);
 733             dy02             = _mm_sub_ps(iy0,jy2);
 734             dz02             = _mm_sub_ps(iz0,jz2);
 735             dx10             = _mm_sub_ps(ix1,jx0);
 736             dy10             = _mm_sub_ps(iy1,jy0);
 737             dz10             = _mm_sub_ps(iz1,jz0);
 738             dx11             = _mm_sub_ps(ix1,jx1);
 739             dy11             = _mm_sub_ps(iy1,jy1);
 740             dz11             = _mm_sub_ps(iz1,jz1);
 741             dx12             = _mm_sub_ps(ix1,jx2);
 742             dy12             = _mm_sub_ps(iy1,jy2);
 743             dz12             = _mm_sub_ps(iz1,jz2);
 744             dx20             = _mm_sub_ps(ix2,jx0);
 745             dy20             = _mm_sub_ps(iy2,jy0);
 746             dz20             = _mm_sub_ps(iz2,jz0);
 747             dx21             = _mm_sub_ps(ix2,jx1);
 748             dy21             = _mm_sub_ps(iy2,jy1);
 749             dz21             = _mm_sub_ps(iz2,jz1);
 750             dx22             = _mm_sub_ps(ix2,jx2);
 751             dy22             = _mm_sub_ps(iy2,jy2);
 752             dz22             = _mm_sub_ps(iz2,jz2);
 753
 754             /* Calculate squared distance and things based on it */
 755             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 756             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 757             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 758             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 759             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 760             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 761             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 762             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 763             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 764
 765             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 766             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 767             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 768             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 769             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 770             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 771             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 772             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 773             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 774
 775             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 776
 777             fjx0             = _mm_setzero_ps();
 778             fjy0             = _mm_setzero_ps();
 779             fjz0             = _mm_setzero_ps();
 780             fjx1             = _mm_setzero_ps();
 781             fjy1             = _mm_setzero_ps();
 782             fjz1             = _mm_setzero_ps();
 783             fjx2             = _mm_setzero_ps();
 784             fjy2             = _mm_setzero_ps();
 785             fjz2             = _mm_setzero_ps();
 786
 787             /**************************
 788              * CALCULATE INTERACTIONS *
 789              **************************/
 790
 791             r00              = _mm_mul_ps(rsq00,rinv00);
 792             r00              = _mm_andnot_ps(dummy_mask,r00);
 793
 794             /* Calculate table index by multiplying r with table scale and truncate to integer */
 795             rt               = _mm_mul_ps(r00,vftabscale);
 796             vfitab           = _mm_cvttps_epi32(rt);
 797 #ifdef __XOP__
 798             vfeps            = _mm_frcz_ps(rt);
 799 #else
 800             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 801 #endif
 802             twovfeps         = _mm_add_ps(vfeps,vfeps);
 803             vfitab           = _mm_slli_epi32(vfitab,2);
 804
 805             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 806             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 807             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 808             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 809             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 810             _MM_TRANSPOSE4_PS(Y,F,G,H);
 811             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 812             VV               = _mm_macc_ps(vfeps,Fp,Y);
 813             velec            = _mm_mul_ps(qq00,VV);
 814             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 815             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
 816
 817             /* LENNARD-JONES DISPERSION/REPULSION */
 818
 819             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 820             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 821             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 822             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 823             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 824
 825             /* Update potential sum for this i atom from the interaction with this j atom. */
 826             velec            = _mm_andnot_ps(dummy_mask,velec);
 827             velecsum         = _mm_add_ps(velecsum,velec);
 828             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 829             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 830
 831             fscal            = _mm_add_ps(felec,fvdw);
 832
 833             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 834
 835              /* Update vectorial force */
 836             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 837             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 838             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 839
 840             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 841             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 842             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 843
 844             /**************************
 845              * CALCULATE INTERACTIONS *
 846              **************************/
 847
 848             r01              = _mm_mul_ps(rsq01,rinv01);
 849             r01              = _mm_andnot_ps(dummy_mask,r01);
 850
 851             /* Calculate table index by multiplying r with table scale and truncate to integer */
 852             rt               = _mm_mul_ps(r01,vftabscale);
 853             vfitab           = _mm_cvttps_epi32(rt);
 854 #ifdef __XOP__
 855             vfeps            = _mm_frcz_ps(rt);
 856 #else
 857             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 858 #endif
 859             twovfeps         = _mm_add_ps(vfeps,vfeps);
 860             vfitab           = _mm_slli_epi32(vfitab,2);
 861
 862             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 863             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 864             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 865             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 866             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 867             _MM_TRANSPOSE4_PS(Y,F,G,H);
 868             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 869             VV               = _mm_macc_ps(vfeps,Fp,Y);
 870             velec            = _mm_mul_ps(qq01,VV);
 871             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 872             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
 873
 874             /* Update potential sum for this i atom from the interaction with this j atom. */
 875             velec            = _mm_andnot_ps(dummy_mask,velec);
 876             velecsum         = _mm_add_ps(velecsum,velec);
 877
 878             fscal            = felec;
 879
 880             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 881
 882              /* Update vectorial force */
 883             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 884             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 885             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 886
 887             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 888             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 889             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 890
 891             /**************************
 892              * CALCULATE INTERACTIONS *
 893              **************************/
 894
 895             r02              = _mm_mul_ps(rsq02,rinv02);
 896             r02              = _mm_andnot_ps(dummy_mask,r02);
 897
 898             /* Calculate table index by multiplying r with table scale and truncate to integer */
 899             rt               = _mm_mul_ps(r02,vftabscale);
 900             vfitab           = _mm_cvttps_epi32(rt);
 901 #ifdef __XOP__
 902             vfeps            = _mm_frcz_ps(rt);
 903 #else
 904             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 905 #endif
 906             twovfeps         = _mm_add_ps(vfeps,vfeps);
 907             vfitab           = _mm_slli_epi32(vfitab,2);
 908
 909             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 910             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 911             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 912             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 913             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 914             _MM_TRANSPOSE4_PS(Y,F,G,H);
 915             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 916             VV               = _mm_macc_ps(vfeps,Fp,Y);
 917             velec            = _mm_mul_ps(qq02,VV);
 918             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 919             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
 920
 921             /* Update potential sum for this i atom from the interaction with this j atom. */
 922             velec            = _mm_andnot_ps(dummy_mask,velec);
 923             velecsum         = _mm_add_ps(velecsum,velec);
 924
 925             fscal            = felec;
 926
 927             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 928
 929              /* Update vectorial force */
 930             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 931             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 932             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 933
 934             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 935             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 936             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 937
 938             /**************************
 939              * CALCULATE INTERACTIONS *
 940              **************************/
 941
 942             r10              = _mm_mul_ps(rsq10,rinv10);
 943             r10              = _mm_andnot_ps(dummy_mask,r10);
 944
 945             /* Calculate table index by multiplying r with table scale and truncate to integer */
 946             rt               = _mm_mul_ps(r10,vftabscale);
 947             vfitab           = _mm_cvttps_epi32(rt);
 948 #ifdef __XOP__
 949             vfeps            = _mm_frcz_ps(rt);
 950 #else
 951             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 952 #endif
 953             twovfeps         = _mm_add_ps(vfeps,vfeps);
 954             vfitab           = _mm_slli_epi32(vfitab,2);
 955
 956             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 957             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 958             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 959             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 960             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 961             _MM_TRANSPOSE4_PS(Y,F,G,H);
 962             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 963             VV               = _mm_macc_ps(vfeps,Fp,Y);
 964             velec            = _mm_mul_ps(qq10,VV);
 965             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 966             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
 967
 968             /* Update potential sum for this i atom from the interaction with this j atom. */
 969             velec            = _mm_andnot_ps(dummy_mask,velec);
 970             velecsum         = _mm_add_ps(velecsum,velec);
 971
 972             fscal            = felec;
 973
 974             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 975
 976              /* Update vectorial force */
 977             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 978             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 979             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 980
 981             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 982             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 983             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 984
 985             /**************************
 986              * CALCULATE INTERACTIONS *
 987              **************************/
 988
 989             r11              = _mm_mul_ps(rsq11,rinv11);
 990             r11              = _mm_andnot_ps(dummy_mask,r11);
 991
 992             /* Calculate table index by multiplying r with table scale and truncate to integer */
 993             rt               = _mm_mul_ps(r11,vftabscale);
 994             vfitab           = _mm_cvttps_epi32(rt);
 995 #ifdef __XOP__
 996             vfeps            = _mm_frcz_ps(rt);
 997 #else
 998             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 999 #endif
1000             twovfeps         = _mm_add_ps(vfeps,vfeps);
1001             vfitab           = _mm_slli_epi32(vfitab,2);
1002
1003             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1004             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1005             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1006             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1007             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1008             _MM_TRANSPOSE4_PS(Y,F,G,H);
1009             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1010             VV               = _mm_macc_ps(vfeps,Fp,Y);
1011             velec            = _mm_mul_ps(qq11,VV);
1012             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1013             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
1014
1015             /* Update potential sum for this i atom from the interaction with this j atom. */
1016             velec            = _mm_andnot_ps(dummy_mask,velec);
1017             velecsum         = _mm_add_ps(velecsum,velec);
1018
1019             fscal            = felec;
1020
1021             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1022
1023              /* Update vectorial force */
1024             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1025             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1026             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1027
1028             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1029             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1030             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1031
1032             /**************************
1033              * CALCULATE INTERACTIONS *
1034              **************************/
1035
1036             r12              = _mm_mul_ps(rsq12,rinv12);
1037             r12              = _mm_andnot_ps(dummy_mask,r12);
1038
1039             /* Calculate table index by multiplying r with table scale and truncate to integer */
1040             rt               = _mm_mul_ps(r12,vftabscale);
1041             vfitab           = _mm_cvttps_epi32(rt);
1042 #ifdef __XOP__
1043             vfeps            = _mm_frcz_ps(rt);
1044 #else
1045             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1046 #endif
1047             twovfeps         = _mm_add_ps(vfeps,vfeps);
1048             vfitab           = _mm_slli_epi32(vfitab,2);
1049
1050             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1051             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1052             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1053             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1054             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1055             _MM_TRANSPOSE4_PS(Y,F,G,H);
1056             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1057             VV               = _mm_macc_ps(vfeps,Fp,Y);
1058             velec            = _mm_mul_ps(qq12,VV);
1059             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1060             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
1061
1062             /* Update potential sum for this i atom from the interaction with this j atom. */
1063             velec            = _mm_andnot_ps(dummy_mask,velec);
1064             velecsum         = _mm_add_ps(velecsum,velec);
1065
1066             fscal            = felec;
1067
1068             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1069
1070              /* Update vectorial force */
1071             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1072             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1073             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1074
1075             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1076             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1077             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1078
1079             /**************************
1080              * CALCULATE INTERACTIONS *
1081              **************************/
1082
1083             r20              = _mm_mul_ps(rsq20,rinv20);
1084             r20              = _mm_andnot_ps(dummy_mask,r20);
1085
1086             /* Calculate table index by multiplying r with table scale and truncate to integer */
1087             rt               = _mm_mul_ps(r20,vftabscale);
1088             vfitab           = _mm_cvttps_epi32(rt);
1089 #ifdef __XOP__
1090             vfeps            = _mm_frcz_ps(rt);
1091 #else
1092             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1093 #endif
1094             twovfeps         = _mm_add_ps(vfeps,vfeps);
1095             vfitab           = _mm_slli_epi32(vfitab,2);
1096
1097             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1098             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1099             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1100             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1101             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1102             _MM_TRANSPOSE4_PS(Y,F,G,H);
1103             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1104             VV               = _mm_macc_ps(vfeps,Fp,Y);
1105             velec            = _mm_mul_ps(qq20,VV);
1106             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1107             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1108
1109             /* Update potential sum for this i atom from the interaction with this j atom. */
1110             velec            = _mm_andnot_ps(dummy_mask,velec);
1111             velecsum         = _mm_add_ps(velecsum,velec);
1112
1113             fscal            = felec;
1114
1115             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1116
1117              /* Update vectorial force */
1118             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1119             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1120             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1121
1122             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1123             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1124             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1125
1126             /**************************
1127              * CALCULATE INTERACTIONS *
1128              **************************/
1129
1130             r21              = _mm_mul_ps(rsq21,rinv21);
1131             r21              = _mm_andnot_ps(dummy_mask,r21);
1132
1133             /* Calculate table index by multiplying r with table scale and truncate to integer */
1134             rt               = _mm_mul_ps(r21,vftabscale);
1135             vfitab           = _mm_cvttps_epi32(rt);
1136 #ifdef __XOP__
1137             vfeps            = _mm_frcz_ps(rt);
1138 #else
1139             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1140 #endif
1141             twovfeps         = _mm_add_ps(vfeps,vfeps);
1142             vfitab           = _mm_slli_epi32(vfitab,2);
1143
1144             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1145             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1146             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1147             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1148             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1149             _MM_TRANSPOSE4_PS(Y,F,G,H);
1150             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1151             VV               = _mm_macc_ps(vfeps,Fp,Y);
1152             velec            = _mm_mul_ps(qq21,VV);
1153             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1154             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
1155
1156             /* Update potential sum for this i atom from the interaction with this j atom. */
1157             velec            = _mm_andnot_ps(dummy_mask,velec);
1158             velecsum         = _mm_add_ps(velecsum,velec);
1159
1160             fscal            = felec;
1161
1162             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1163
1164              /* Update vectorial force */
1165             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1166             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1167             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1168
1169             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1170             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1171             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1172
1173             /**************************
1174              * CALCULATE INTERACTIONS *
1175              **************************/
1176
1177             r22              = _mm_mul_ps(rsq22,rinv22);
1178             r22              = _mm_andnot_ps(dummy_mask,r22);
1179
1180             /* Calculate table index by multiplying r with table scale and truncate to integer */
1181             rt               = _mm_mul_ps(r22,vftabscale);
1182             vfitab           = _mm_cvttps_epi32(rt);
1183 #ifdef __XOP__
1184             vfeps            = _mm_frcz_ps(rt);
1185 #else
1186             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1187 #endif
1188             twovfeps         = _mm_add_ps(vfeps,vfeps);
1189             vfitab           = _mm_slli_epi32(vfitab,2);
1190
1191             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1192             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1193             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1194             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1195             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1196             _MM_TRANSPOSE4_PS(Y,F,G,H);
1197             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1198             VV               = _mm_macc_ps(vfeps,Fp,Y);
1199             velec            = _mm_mul_ps(qq22,VV);
1200             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1201             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
1202
1203             /* Update potential sum for this i atom from the interaction with this j atom. */
1204             velec            = _mm_andnot_ps(dummy_mask,velec);
1205             velecsum         = _mm_add_ps(velecsum,velec);
1206
1207             fscal            = felec;
1208
1209             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1210
1211              /* Update vectorial force */
1212             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1213             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1214             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1215
1216             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1217             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1218             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1219
1220             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1221             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1222             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1223             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1224
1225             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1226                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1227
1228             /* Inner loop uses 436 flops */
1229         }
1230
1231         /* End of innermost loop */
1232
1233         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1234                                               f+i_coord_offset,fshift+i_shift_offset);
1235
1236         ggid                        = gid[iidx];
1237         /* Update potential energies */
1238         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1239         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1240
1241         /* Increment number of inner iterations */
1242         inneriter                  += j_index_end - j_index_start;
1243
1244         /* Outer loop uses 20 flops */
1245     }
1246
1247     /* Increment number of outer iterations */
1248     outeriter        += nri;
1249
1250     /* Update outer/inner flops */
1251
1252     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*436);
1253 }
1254 /*
1255  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_F_avx_128_fma_single
1256  * Electrostatics interaction: CubicSplineTable
1257  * VdW interaction:            LennardJones
1258  * Geometry:                   Water3-Water3
1259  * Calculate force/pot:        Force
1260  */
1261 void
1262 nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_F_avx_128_fma_single
1263                     (t_nblist                    * gmx_restrict       nlist,
1264                      rvec                        * gmx_restrict          xx,
1265                      rvec                        * gmx_restrict          ff,
1266                      t_forcerec                  * gmx_restrict          fr,
1267                      t_mdatoms                   * gmx_restrict     mdatoms,
1268                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1269                      t_nrnb                      * gmx_restrict        nrnb)
1270 {
1271     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1272      * just 0 for non-waters.
1273      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1274      * jnr indices corresponding to data put in the four positions in the SIMD register.
1275      */
1276     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
1277     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1278     int              jnrA,jnrB,jnrC,jnrD;
1279     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1280     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1281     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
1282     real             rcutoff_scalar;
1283     real             *shiftvec,*fshift,*x,*f;
1284     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1285     real             scratch[4*DIM];
1286     __m128           fscal,rcutoff,rcutoff2,jidxall;
1287     int              vdwioffset0;
1288     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1289     int              vdwioffset1;
1290     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1291     int              vdwioffset2;
1292     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1293     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1294     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1295     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1296     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1297     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1298     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1299     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1300     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1301     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1302     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1303     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1304     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1305     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1306     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1307     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1308     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
1309     real             *charge;
1310     int              nvdwtype;
1311     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1312     int              *vdwtype;
1313     real             *vdwparam;
1314     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
1315     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
1316     __m128i          vfitab;
1317     __m128i          ifour       = _mm_set1_epi32(4);
1318     __m128           rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
1319     real             *vftab;
1320     __m128           dummy_mask,cutoff_mask;
1321     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1322     __m128           one     = _mm_set1_ps(1.0);
1323     __m128           two     = _mm_set1_ps(2.0);
1324     x                = xx[0];
1325     f                = ff[0];
1326
1327     nri              = nlist->nri;
1328     iinr             = nlist->iinr;
1329     jindex           = nlist->jindex;
1330     jjnr             = nlist->jjnr;
1331     shiftidx         = nlist->shift;
1332     gid              = nlist->gid;
1333     shiftvec         = fr->shift_vec[0];
1334     fshift           = fr->fshift[0];
1335     facel            = _mm_set1_ps(fr->epsfac);
1336     charge           = mdatoms->chargeA;
1337     nvdwtype         = fr->ntype;
1338     vdwparam         = fr->nbfp;
1339     vdwtype          = mdatoms->typeA;
1340
1341     vftab            = kernel_data->table_elec->data;
1342     vftabscale       = _mm_set1_ps(kernel_data->table_elec->scale);
1343
1344     /* Setup water-specific parameters */
1345     inr              = nlist->iinr[0];
1346     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1347     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1348     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1349     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
1350
1351     jq0              = _mm_set1_ps(charge[inr+0]);
1352     jq1              = _mm_set1_ps(charge[inr+1]);
1353     jq2              = _mm_set1_ps(charge[inr+2]);
1354     vdwjidx0A        = 2*vdwtype[inr+0];
1355     qq00             = _mm_mul_ps(iq0,jq0);
1356     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1357     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1358     qq01             = _mm_mul_ps(iq0,jq1);
1359     qq02             = _mm_mul_ps(iq0,jq2);
1360     qq10             = _mm_mul_ps(iq1,jq0);
1361     qq11             = _mm_mul_ps(iq1,jq1);
1362     qq12             = _mm_mul_ps(iq1,jq2);
1363     qq20             = _mm_mul_ps(iq2,jq0);
1364     qq21             = _mm_mul_ps(iq2,jq1);
1365     qq22             = _mm_mul_ps(iq2,jq2);
1366
1367     /* Avoid stupid compiler warnings */
1368     jnrA = jnrB = jnrC = jnrD = 0;
1369     j_coord_offsetA = 0;
1370     j_coord_offsetB = 0;
1371     j_coord_offsetC = 0;
1372     j_coord_offsetD = 0;
1373
1374     outeriter        = 0;
1375     inneriter        = 0;
1376
1377     for(iidx=0;iidx<4*DIM;iidx++)
1378     {
1379         scratch[iidx] = 0.0;
1380     }
1381
1382     /* Start outer loop over neighborlists */
1383     for(iidx=0; iidx<nri; iidx++)
1384     {
1385         /* Load shift vector for this list */
1386         i_shift_offset   = DIM*shiftidx[iidx];
1387
1388         /* Load limits for loop over neighbors */
1389         j_index_start    = jindex[iidx];
1390         j_index_end      = jindex[iidx+1];
1391
1392         /* Get outer coordinate index */
1393         inr              = iinr[iidx];
1394         i_coord_offset   = DIM*inr;
1395
1396         /* Load i particle coords and add shift vector */
1397         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1398                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1399
1400         fix0             = _mm_setzero_ps();
1401         fiy0             = _mm_setzero_ps();
1402         fiz0             = _mm_setzero_ps();
1403         fix1             = _mm_setzero_ps();
1404         fiy1             = _mm_setzero_ps();
1405         fiz1             = _mm_setzero_ps();
1406         fix2             = _mm_setzero_ps();
1407         fiy2             = _mm_setzero_ps();
1408         fiz2             = _mm_setzero_ps();
1409
1410         /* Start inner kernel loop */
1411         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1412         {
1413
1414             /* Get j neighbor index, and coordinate index */
1415             jnrA             = jjnr[jidx];
1416             jnrB             = jjnr[jidx+1];
1417             jnrC             = jjnr[jidx+2];
1418             jnrD             = jjnr[jidx+3];
1419             j_coord_offsetA  = DIM*jnrA;
1420             j_coord_offsetB  = DIM*jnrB;
1421             j_coord_offsetC  = DIM*jnrC;
1422             j_coord_offsetD  = DIM*jnrD;
1423
1424             /* load j atom coordinates */
1425             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1426                                               x+j_coord_offsetC,x+j_coord_offsetD,
1427                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1428
1429             /* Calculate displacement vector */
1430             dx00             = _mm_sub_ps(ix0,jx0);
1431             dy00             = _mm_sub_ps(iy0,jy0);
1432             dz00             = _mm_sub_ps(iz0,jz0);
1433             dx01             = _mm_sub_ps(ix0,jx1);
1434             dy01             = _mm_sub_ps(iy0,jy1);
1435             dz01             = _mm_sub_ps(iz0,jz1);
1436             dx02             = _mm_sub_ps(ix0,jx2);
1437             dy02             = _mm_sub_ps(iy0,jy2);
1438             dz02             = _mm_sub_ps(iz0,jz2);
1439             dx10             = _mm_sub_ps(ix1,jx0);
1440             dy10             = _mm_sub_ps(iy1,jy0);
1441             dz10             = _mm_sub_ps(iz1,jz0);
1442             dx11             = _mm_sub_ps(ix1,jx1);
1443             dy11             = _mm_sub_ps(iy1,jy1);
1444             dz11             = _mm_sub_ps(iz1,jz1);
1445             dx12             = _mm_sub_ps(ix1,jx2);
1446             dy12             = _mm_sub_ps(iy1,jy2);
1447             dz12             = _mm_sub_ps(iz1,jz2);
1448             dx20             = _mm_sub_ps(ix2,jx0);
1449             dy20             = _mm_sub_ps(iy2,jy0);
1450             dz20             = _mm_sub_ps(iz2,jz0);
1451             dx21             = _mm_sub_ps(ix2,jx1);
1452             dy21             = _mm_sub_ps(iy2,jy1);
1453             dz21             = _mm_sub_ps(iz2,jz1);
1454             dx22             = _mm_sub_ps(ix2,jx2);
1455             dy22             = _mm_sub_ps(iy2,jy2);
1456             dz22             = _mm_sub_ps(iz2,jz2);
1457
1458             /* Calculate squared distance and things based on it */
1459             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1460             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1461             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1462             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1463             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1464             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1465             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1466             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1467             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1468
1469             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1470             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1471             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1472             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1473             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1474             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1475             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1476             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1477             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1478
1479             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1480
1481             fjx0             = _mm_setzero_ps();
1482             fjy0             = _mm_setzero_ps();
1483             fjz0             = _mm_setzero_ps();
1484             fjx1             = _mm_setzero_ps();
1485             fjy1             = _mm_setzero_ps();
1486             fjz1             = _mm_setzero_ps();
1487             fjx2             = _mm_setzero_ps();
1488             fjy2             = _mm_setzero_ps();
1489             fjz2             = _mm_setzero_ps();
1490
1491             /**************************
1492              * CALCULATE INTERACTIONS *
1493              **************************/
1494
1495             r00              = _mm_mul_ps(rsq00,rinv00);
1496
1497             /* Calculate table index by multiplying r with table scale and truncate to integer */
1498             rt               = _mm_mul_ps(r00,vftabscale);
1499             vfitab           = _mm_cvttps_epi32(rt);
1500 #ifdef __XOP__
1501             vfeps            = _mm_frcz_ps(rt);
1502 #else
1503             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1504 #endif
1505             twovfeps         = _mm_add_ps(vfeps,vfeps);
1506             vfitab           = _mm_slli_epi32(vfitab,2);
1507
1508             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1509             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1510             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1511             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1512             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1513             _MM_TRANSPOSE4_PS(Y,F,G,H);
1514             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1515             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1516             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
1517
1518             /* LENNARD-JONES DISPERSION/REPULSION */
1519
1520             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1521             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1522
1523             fscal            = _mm_add_ps(felec,fvdw);
1524
1525              /* Update vectorial force */
1526             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1527             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1528             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1529
1530             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1531             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1532             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1533
1534             /**************************
1535              * CALCULATE INTERACTIONS *
1536              **************************/
1537
1538             r01              = _mm_mul_ps(rsq01,rinv01);
1539
1540             /* Calculate table index by multiplying r with table scale and truncate to integer */
1541             rt               = _mm_mul_ps(r01,vftabscale);
1542             vfitab           = _mm_cvttps_epi32(rt);
1543 #ifdef __XOP__
1544             vfeps            = _mm_frcz_ps(rt);
1545 #else
1546             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1547 #endif
1548             twovfeps         = _mm_add_ps(vfeps,vfeps);
1549             vfitab           = _mm_slli_epi32(vfitab,2);
1550
1551             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1552             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1553             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1554             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1555             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1556             _MM_TRANSPOSE4_PS(Y,F,G,H);
1557             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1558             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1559             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
1560
1561             fscal            = felec;
1562
1563              /* Update vectorial force */
1564             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1565             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1566             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1567
1568             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1569             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1570             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1571
1572             /**************************
1573              * CALCULATE INTERACTIONS *
1574              **************************/
1575
1576             r02              = _mm_mul_ps(rsq02,rinv02);
1577
1578             /* Calculate table index by multiplying r with table scale and truncate to integer */
1579             rt               = _mm_mul_ps(r02,vftabscale);
1580             vfitab           = _mm_cvttps_epi32(rt);
1581 #ifdef __XOP__
1582             vfeps            = _mm_frcz_ps(rt);
1583 #else
1584             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1585 #endif
1586             twovfeps         = _mm_add_ps(vfeps,vfeps);
1587             vfitab           = _mm_slli_epi32(vfitab,2);
1588
1589             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1590             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1591             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1592             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1593             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1594             _MM_TRANSPOSE4_PS(Y,F,G,H);
1595             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1596             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1597             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
1598
1599             fscal            = felec;
1600
1601              /* Update vectorial force */
1602             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1603             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1604             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1605
1606             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1607             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1608             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1609
1610             /**************************
1611              * CALCULATE INTERACTIONS *
1612              **************************/
1613
1614             r10              = _mm_mul_ps(rsq10,rinv10);
1615
1616             /* Calculate table index by multiplying r with table scale and truncate to integer */
1617             rt               = _mm_mul_ps(r10,vftabscale);
1618             vfitab           = _mm_cvttps_epi32(rt);
1619 #ifdef __XOP__
1620             vfeps            = _mm_frcz_ps(rt);
1621 #else
1622             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1623 #endif
1624             twovfeps         = _mm_add_ps(vfeps,vfeps);
1625             vfitab           = _mm_slli_epi32(vfitab,2);
1626
1627             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1628             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1629             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1630             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1631             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1632             _MM_TRANSPOSE4_PS(Y,F,G,H);
1633             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1634             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1635             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
1636
1637             fscal            = felec;
1638
1639              /* Update vectorial force */
1640             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1641             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1642             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1643
1644             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1645             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1646             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1647
1648             /**************************
1649              * CALCULATE INTERACTIONS *
1650              **************************/
1651
1652             r11              = _mm_mul_ps(rsq11,rinv11);
1653
1654             /* Calculate table index by multiplying r with table scale and truncate to integer */
1655             rt               = _mm_mul_ps(r11,vftabscale);
1656             vfitab           = _mm_cvttps_epi32(rt);
1657 #ifdef __XOP__
1658             vfeps            = _mm_frcz_ps(rt);
1659 #else
1660             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1661 #endif
1662             twovfeps         = _mm_add_ps(vfeps,vfeps);
1663             vfitab           = _mm_slli_epi32(vfitab,2);
1664
1665             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1666             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1667             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1668             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1669             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1670             _MM_TRANSPOSE4_PS(Y,F,G,H);
1671             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1672             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1673             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
1674
1675             fscal            = felec;
1676
1677              /* Update vectorial force */
1678             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1679             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1680             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1681
1682             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1683             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1684             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1685
1686             /**************************
1687              * CALCULATE INTERACTIONS *
1688              **************************/
1689
1690             r12              = _mm_mul_ps(rsq12,rinv12);
1691
1692             /* Calculate table index by multiplying r with table scale and truncate to integer */
1693             rt               = _mm_mul_ps(r12,vftabscale);
1694             vfitab           = _mm_cvttps_epi32(rt);
1695 #ifdef __XOP__
1696             vfeps            = _mm_frcz_ps(rt);
1697 #else
1698             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1699 #endif
1700             twovfeps         = _mm_add_ps(vfeps,vfeps);
1701             vfitab           = _mm_slli_epi32(vfitab,2);
1702
1703             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1704             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1705             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1706             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1707             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1708             _MM_TRANSPOSE4_PS(Y,F,G,H);
1709             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1710             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1711             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
1712
1713             fscal            = felec;
1714
1715              /* Update vectorial force */
1716             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1717             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1718             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1719
1720             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1721             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1722             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1723
1724             /**************************
1725              * CALCULATE INTERACTIONS *
1726              **************************/
1727
1728             r20              = _mm_mul_ps(rsq20,rinv20);
1729
1730             /* Calculate table index by multiplying r with table scale and truncate to integer */
1731             rt               = _mm_mul_ps(r20,vftabscale);
1732             vfitab           = _mm_cvttps_epi32(rt);
1733 #ifdef __XOP__
1734             vfeps            = _mm_frcz_ps(rt);
1735 #else
1736             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1737 #endif
1738             twovfeps         = _mm_add_ps(vfeps,vfeps);
1739             vfitab           = _mm_slli_epi32(vfitab,2);
1740
1741             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1742             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1743             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1744             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1745             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1746             _MM_TRANSPOSE4_PS(Y,F,G,H);
1747             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1748             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1749             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1750
1751             fscal            = felec;
1752
1753              /* Update vectorial force */
1754             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1755             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1756             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1757
1758             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1759             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1760             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1761
1762             /**************************
1763              * CALCULATE INTERACTIONS *
1764              **************************/
1765
1766             r21              = _mm_mul_ps(rsq21,rinv21);
1767
1768             /* Calculate table index by multiplying r with table scale and truncate to integer */
1769             rt               = _mm_mul_ps(r21,vftabscale);
1770             vfitab           = _mm_cvttps_epi32(rt);
1771 #ifdef __XOP__
1772             vfeps            = _mm_frcz_ps(rt);
1773 #else
1774             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1775 #endif
1776             twovfeps         = _mm_add_ps(vfeps,vfeps);
1777             vfitab           = _mm_slli_epi32(vfitab,2);
1778
1779             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1780             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1781             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1782             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1783             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1784             _MM_TRANSPOSE4_PS(Y,F,G,H);
1785             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1786             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1787             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
1788
1789             fscal            = felec;
1790
1791              /* Update vectorial force */
1792             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1793             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1794             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1795
1796             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1797             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1798             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1799
1800             /**************************
1801              * CALCULATE INTERACTIONS *
1802              **************************/
1803
1804             r22              = _mm_mul_ps(rsq22,rinv22);
1805
1806             /* Calculate table index by multiplying r with table scale and truncate to integer */
1807             rt               = _mm_mul_ps(r22,vftabscale);
1808             vfitab           = _mm_cvttps_epi32(rt);
1809 #ifdef __XOP__
1810             vfeps            = _mm_frcz_ps(rt);
1811 #else
1812             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1813 #endif
1814             twovfeps         = _mm_add_ps(vfeps,vfeps);
1815             vfitab           = _mm_slli_epi32(vfitab,2);
1816
1817             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1818             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1819             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1820             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1821             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1822             _MM_TRANSPOSE4_PS(Y,F,G,H);
1823             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1824             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1825             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
1826
1827             fscal            = felec;
1828
1829              /* Update vectorial force */
1830             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1831             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1832             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1833
1834             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1835             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1836             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1837
1838             fjptrA             = f+j_coord_offsetA;
1839             fjptrB             = f+j_coord_offsetB;
1840             fjptrC             = f+j_coord_offsetC;
1841             fjptrD             = f+j_coord_offsetD;
1842
1843             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1844                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1845
1846             /* Inner loop uses 386 flops */
1847         }
1848
1849         if(jidx<j_index_end)
1850         {
1851
1852             /* Get j neighbor index, and coordinate index */
1853             jnrlistA         = jjnr[jidx];
1854             jnrlistB         = jjnr[jidx+1];
1855             jnrlistC         = jjnr[jidx+2];
1856             jnrlistD         = jjnr[jidx+3];
1857             /* Sign of each element will be negative for non-real atoms.
1858              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1859              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1860              */
1861             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1862             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
1863             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
1864             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1865             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1866             j_coord_offsetA  = DIM*jnrA;
1867             j_coord_offsetB  = DIM*jnrB;
1868             j_coord_offsetC  = DIM*jnrC;
1869             j_coord_offsetD  = DIM*jnrD;
1870
1871             /* load j atom coordinates */
1872             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1873                                               x+j_coord_offsetC,x+j_coord_offsetD,
1874                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1875
1876             /* Calculate displacement vector */
1877             dx00             = _mm_sub_ps(ix0,jx0);
1878             dy00             = _mm_sub_ps(iy0,jy0);
1879             dz00             = _mm_sub_ps(iz0,jz0);
1880             dx01             = _mm_sub_ps(ix0,jx1);
1881             dy01             = _mm_sub_ps(iy0,jy1);
1882             dz01             = _mm_sub_ps(iz0,jz1);
1883             dx02             = _mm_sub_ps(ix0,jx2);
1884             dy02             = _mm_sub_ps(iy0,jy2);
1885             dz02             = _mm_sub_ps(iz0,jz2);
1886             dx10             = _mm_sub_ps(ix1,jx0);
1887             dy10             = _mm_sub_ps(iy1,jy0);
1888             dz10             = _mm_sub_ps(iz1,jz0);
1889             dx11             = _mm_sub_ps(ix1,jx1);
1890             dy11             = _mm_sub_ps(iy1,jy1);
1891             dz11             = _mm_sub_ps(iz1,jz1);
1892             dx12             = _mm_sub_ps(ix1,jx2);
1893             dy12             = _mm_sub_ps(iy1,jy2);
1894             dz12             = _mm_sub_ps(iz1,jz2);
1895             dx20             = _mm_sub_ps(ix2,jx0);
1896             dy20             = _mm_sub_ps(iy2,jy0);
1897             dz20             = _mm_sub_ps(iz2,jz0);
1898             dx21             = _mm_sub_ps(ix2,jx1);
1899             dy21             = _mm_sub_ps(iy2,jy1);
1900             dz21             = _mm_sub_ps(iz2,jz1);
1901             dx22             = _mm_sub_ps(ix2,jx2);
1902             dy22             = _mm_sub_ps(iy2,jy2);
1903             dz22             = _mm_sub_ps(iz2,jz2);
1904
1905             /* Calculate squared distance and things based on it */
1906             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1907             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1908             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1909             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1910             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1911             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1912             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1913             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1914             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1915
1916             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1917             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1918             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1919             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1920             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1921             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1922             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1923             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1924             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1925
1926             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1927
1928             fjx0             = _mm_setzero_ps();
1929             fjy0             = _mm_setzero_ps();
1930             fjz0             = _mm_setzero_ps();
1931             fjx1             = _mm_setzero_ps();
1932             fjy1             = _mm_setzero_ps();
1933             fjz1             = _mm_setzero_ps();
1934             fjx2             = _mm_setzero_ps();
1935             fjy2             = _mm_setzero_ps();
1936             fjz2             = _mm_setzero_ps();
1937
1938             /**************************
1939              * CALCULATE INTERACTIONS *
1940              **************************/
1941
1942             r00              = _mm_mul_ps(rsq00,rinv00);
1943             r00              = _mm_andnot_ps(dummy_mask,r00);
1944
1945             /* Calculate table index by multiplying r with table scale and truncate to integer */
1946             rt               = _mm_mul_ps(r00,vftabscale);
1947             vfitab           = _mm_cvttps_epi32(rt);
1948 #ifdef __XOP__
1949             vfeps            = _mm_frcz_ps(rt);
1950 #else
1951             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1952 #endif
1953             twovfeps         = _mm_add_ps(vfeps,vfeps);
1954             vfitab           = _mm_slli_epi32(vfitab,2);
1955
1956             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1957             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1958             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1959             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1960             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1961             _MM_TRANSPOSE4_PS(Y,F,G,H);
1962             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1963             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1964             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
1965
1966             /* LENNARD-JONES DISPERSION/REPULSION */
1967
1968             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1969             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1970
1971             fscal            = _mm_add_ps(felec,fvdw);
1972
1973             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1974
1975              /* Update vectorial force */
1976             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1977             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1978             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1979
1980             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1981             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1982             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1983
1984             /**************************
1985              * CALCULATE INTERACTIONS *
1986              **************************/
1987
1988             r01              = _mm_mul_ps(rsq01,rinv01);
1989             r01              = _mm_andnot_ps(dummy_mask,r01);
1990
1991             /* Calculate table index by multiplying r with table scale and truncate to integer */
1992             rt               = _mm_mul_ps(r01,vftabscale);
1993             vfitab           = _mm_cvttps_epi32(rt);
1994 #ifdef __XOP__
1995             vfeps            = _mm_frcz_ps(rt);
1996 #else
1997             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1998 #endif
1999             twovfeps         = _mm_add_ps(vfeps,vfeps);
2000             vfitab           = _mm_slli_epi32(vfitab,2);
2001
2002             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2003             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2004             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2005             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2006             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2007             _MM_TRANSPOSE4_PS(Y,F,G,H);
2008             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2009             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2010             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
2011
2012             fscal            = felec;
2013
2014             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2015
2016              /* Update vectorial force */
2017             fix0             = _mm_macc_ps(dx01,fscal,fix0);
2018             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
2019             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
2020
2021             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
2022             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
2023             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
2024
2025             /**************************
2026              * CALCULATE INTERACTIONS *
2027              **************************/
2028
2029             r02              = _mm_mul_ps(rsq02,rinv02);
2030             r02              = _mm_andnot_ps(dummy_mask,r02);
2031
2032             /* Calculate table index by multiplying r with table scale and truncate to integer */
2033             rt               = _mm_mul_ps(r02,vftabscale);
2034             vfitab           = _mm_cvttps_epi32(rt);
2035 #ifdef __XOP__
2036             vfeps            = _mm_frcz_ps(rt);
2037 #else
2038             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2039 #endif
2040             twovfeps         = _mm_add_ps(vfeps,vfeps);
2041             vfitab           = _mm_slli_epi32(vfitab,2);
2042
2043             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2044             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2045             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2046             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2047             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2048             _MM_TRANSPOSE4_PS(Y,F,G,H);
2049             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2050             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2051             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
2052
2053             fscal            = felec;
2054
2055             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2056
2057              /* Update vectorial force */
2058             fix0             = _mm_macc_ps(dx02,fscal,fix0);
2059             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
2060             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
2061
2062             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
2063             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
2064             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
2065
2066             /**************************
2067              * CALCULATE INTERACTIONS *
2068              **************************/
2069
2070             r10              = _mm_mul_ps(rsq10,rinv10);
2071             r10              = _mm_andnot_ps(dummy_mask,r10);
2072
2073             /* Calculate table index by multiplying r with table scale and truncate to integer */
2074             rt               = _mm_mul_ps(r10,vftabscale);
2075             vfitab           = _mm_cvttps_epi32(rt);
2076 #ifdef __XOP__
2077             vfeps            = _mm_frcz_ps(rt);
2078 #else
2079             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2080 #endif
2081             twovfeps         = _mm_add_ps(vfeps,vfeps);
2082             vfitab           = _mm_slli_epi32(vfitab,2);
2083
2084             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2085             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2086             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2087             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2088             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2089             _MM_TRANSPOSE4_PS(Y,F,G,H);
2090             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2091             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2092             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
2093
2094             fscal            = felec;
2095
2096             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2097
2098              /* Update vectorial force */
2099             fix1             = _mm_macc_ps(dx10,fscal,fix1);
2100             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
2101             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
2102
2103             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
2104             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
2105             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
2106
2107             /**************************
2108              * CALCULATE INTERACTIONS *
2109              **************************/
2110
2111             r11              = _mm_mul_ps(rsq11,rinv11);
2112             r11              = _mm_andnot_ps(dummy_mask,r11);
2113
2114             /* Calculate table index by multiplying r with table scale and truncate to integer */
2115             rt               = _mm_mul_ps(r11,vftabscale);
2116             vfitab           = _mm_cvttps_epi32(rt);
2117 #ifdef __XOP__
2118             vfeps            = _mm_frcz_ps(rt);
2119 #else
2120             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2121 #endif
2122             twovfeps         = _mm_add_ps(vfeps,vfeps);
2123             vfitab           = _mm_slli_epi32(vfitab,2);
2124
2125             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2126             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2127             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2128             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2129             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2130             _MM_TRANSPOSE4_PS(Y,F,G,H);
2131             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2132             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2133             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
2134
2135             fscal            = felec;
2136
2137             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2138
2139              /* Update vectorial force */
2140             fix1             = _mm_macc_ps(dx11,fscal,fix1);
2141             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
2142             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
2143
2144             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
2145             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
2146             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
2147
2148             /**************************
2149              * CALCULATE INTERACTIONS *
2150              **************************/
2151
2152             r12              = _mm_mul_ps(rsq12,rinv12);
2153             r12              = _mm_andnot_ps(dummy_mask,r12);
2154
2155             /* Calculate table index by multiplying r with table scale and truncate to integer */
2156             rt               = _mm_mul_ps(r12,vftabscale);
2157             vfitab           = _mm_cvttps_epi32(rt);
2158 #ifdef __XOP__
2159             vfeps            = _mm_frcz_ps(rt);
2160 #else
2161             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2162 #endif
2163             twovfeps         = _mm_add_ps(vfeps,vfeps);
2164             vfitab           = _mm_slli_epi32(vfitab,2);
2165
2166             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2167             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2168             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2169             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2170             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2171             _MM_TRANSPOSE4_PS(Y,F,G,H);
2172             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2173             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2174             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
2175
2176             fscal            = felec;
2177
2178             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2179
2180              /* Update vectorial force */
2181             fix1             = _mm_macc_ps(dx12,fscal,fix1);
2182             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
2183             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
2184
2185             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
2186             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
2187             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
2188
2189             /**************************
2190              * CALCULATE INTERACTIONS *
2191              **************************/
2192
2193             r20              = _mm_mul_ps(rsq20,rinv20);
2194             r20              = _mm_andnot_ps(dummy_mask,r20);
2195
2196             /* Calculate table index by multiplying r with table scale and truncate to integer */
2197             rt               = _mm_mul_ps(r20,vftabscale);
2198             vfitab           = _mm_cvttps_epi32(rt);
2199 #ifdef __XOP__
2200             vfeps            = _mm_frcz_ps(rt);
2201 #else
2202             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2203 #endif
2204             twovfeps         = _mm_add_ps(vfeps,vfeps);
2205             vfitab           = _mm_slli_epi32(vfitab,2);
2206
2207             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2208             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2209             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2210             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2211             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2212             _MM_TRANSPOSE4_PS(Y,F,G,H);
2213             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2214             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2215             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
2216
2217             fscal            = felec;
2218
2219             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2220
2221              /* Update vectorial force */
2222             fix2             = _mm_macc_ps(dx20,fscal,fix2);
2223             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
2224             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
2225
2226             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
2227             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
2228             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
2229
2230             /**************************
2231              * CALCULATE INTERACTIONS *
2232              **************************/
2233
2234             r21              = _mm_mul_ps(rsq21,rinv21);
2235             r21              = _mm_andnot_ps(dummy_mask,r21);
2236
2237             /* Calculate table index by multiplying r with table scale and truncate to integer */
2238             rt               = _mm_mul_ps(r21,vftabscale);
2239             vfitab           = _mm_cvttps_epi32(rt);
2240 #ifdef __XOP__
2241             vfeps            = _mm_frcz_ps(rt);
2242 #else
2243             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2244 #endif
2245             twovfeps         = _mm_add_ps(vfeps,vfeps);
2246             vfitab           = _mm_slli_epi32(vfitab,2);
2247
2248             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2249             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2250             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2251             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2252             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2253             _MM_TRANSPOSE4_PS(Y,F,G,H);
2254             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2255             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2256             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
2257
2258             fscal            = felec;
2259
2260             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2261
2262              /* Update vectorial force */
2263             fix2             = _mm_macc_ps(dx21,fscal,fix2);
2264             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
2265             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
2266
2267             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
2268             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
2269             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
2270
2271             /**************************
2272              * CALCULATE INTERACTIONS *
2273              **************************/
2274
2275             r22              = _mm_mul_ps(rsq22,rinv22);
2276             r22              = _mm_andnot_ps(dummy_mask,r22);
2277
2278             /* Calculate table index by multiplying r with table scale and truncate to integer */
2279             rt               = _mm_mul_ps(r22,vftabscale);
2280             vfitab           = _mm_cvttps_epi32(rt);
2281 #ifdef __XOP__
2282             vfeps            = _mm_frcz_ps(rt);
2283 #else
2284             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2285 #endif
2286             twovfeps         = _mm_add_ps(vfeps,vfeps);
2287             vfitab           = _mm_slli_epi32(vfitab,2);
2288
2289             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2290             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2291             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2292             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2293             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2294             _MM_TRANSPOSE4_PS(Y,F,G,H);
2295             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2296             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2297             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
2298
2299             fscal            = felec;
2300
2301             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2302
2303              /* Update vectorial force */
2304             fix2             = _mm_macc_ps(dx22,fscal,fix2);
2305             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
2306             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
2307
2308             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
2309             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
2310             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
2311
2312             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
2313             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
2314             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
2315             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
2316
2317             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
2318                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
2319
2320             /* Inner loop uses 395 flops */
2321         }
2322
2323         /* End of innermost loop */
2324
2325         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
2326                                               f+i_coord_offset,fshift+i_shift_offset);
2327
2328         /* Increment number of inner iterations */
2329         inneriter                  += j_index_end - j_index_start;
2330
2331         /* Outer loop uses 18 flops */
2332     }
2333
2334     /* Increment number of outer iterations */
2335     outeriter        += nri;
2336
2337     /* Update outer/inner flops */
2338
2339     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*395);
2340 }