src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_avx_128_fma_single.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014,2015,2017, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
  37  */
  38 #include "gmxpre.h"
  39
  40 #include "config.h"
  41
  42 #include <math.h>
  43
  44 #include "../nb_kernel.h"
  45 #include "gromacs/gmxlib/nrnb.h"
  46
  47 #include "kernelutil_x86_avx_128_fma_single.h"
  48
  49 /*
  50  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  51  * Electrostatics interaction: CubicSplineTable
  52  * VdW interaction:            LennardJones
  53  * Geometry:                   Water3-Water3
  54  * Calculate force/pot:        PotentialAndForce
  55  */
  56 void
  57 nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  58                     (t_nblist                    * gmx_restrict       nlist,
  59                      rvec                        * gmx_restrict          xx,
  60                      rvec                        * gmx_restrict          ff,
  61                      struct t_forcerec           * gmx_restrict          fr,
  62                      t_mdatoms                   * gmx_restrict     mdatoms,
  63                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  64                      t_nrnb                      * gmx_restrict        nrnb)
  65 {
  66     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  67      * just 0 for non-waters.
  68      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  69      * jnr indices corresponding to data put in the four positions in the SIMD register.
  70      */
  71     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  72     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  73     int              jnrA,jnrB,jnrC,jnrD;
  74     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  75     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  76     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  77     real             rcutoff_scalar;
  78     real             *shiftvec,*fshift,*x,*f;
  79     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  80     real             scratch[4*DIM];
  81     __m128           fscal,rcutoff,rcutoff2,jidxall;
  82     int              vdwioffset0;
  83     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  84     int              vdwioffset1;
  85     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  86     int              vdwioffset2;
  87     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  88     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  89     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  90     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
  91     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  92     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
  93     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  94     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  95     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
  96     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
  97     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  98     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
  99     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 100     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 101     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 102     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 103     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 104     real             *charge;
 105     int              nvdwtype;
 106     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 107     int              *vdwtype;
 108     real             *vdwparam;
 109     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 110     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 111     __m128i          vfitab;
 112     __m128i          ifour       = _mm_set1_epi32(4);
 113     __m128           rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
 114     real             *vftab;
 115     __m128           dummy_mask,cutoff_mask;
 116     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 117     __m128           one     = _mm_set1_ps(1.0);
 118     __m128           two     = _mm_set1_ps(2.0);
 119     x                = xx[0];
 120     f                = ff[0];
 121
 122     nri              = nlist->nri;
 123     iinr             = nlist->iinr;
 124     jindex           = nlist->jindex;
 125     jjnr             = nlist->jjnr;
 126     shiftidx         = nlist->shift;
 127     gid              = nlist->gid;
 128     shiftvec         = fr->shift_vec[0];
 129     fshift           = fr->fshift[0];
 130     facel            = _mm_set1_ps(fr->ic->epsfac);
 131     charge           = mdatoms->chargeA;
 132     nvdwtype         = fr->ntype;
 133     vdwparam         = fr->nbfp;
 134     vdwtype          = mdatoms->typeA;
 135
 136     vftab            = kernel_data->table_elec->data;
 137     vftabscale       = _mm_set1_ps(kernel_data->table_elec->scale);
 138
 139     /* Setup water-specific parameters */
 140     inr              = nlist->iinr[0];
 141     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 142     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 143     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 144     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 145
 146     jq0              = _mm_set1_ps(charge[inr+0]);
 147     jq1              = _mm_set1_ps(charge[inr+1]);
 148     jq2              = _mm_set1_ps(charge[inr+2]);
 149     vdwjidx0A        = 2*vdwtype[inr+0];
 150     qq00             = _mm_mul_ps(iq0,jq0);
 151     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 152     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 153     qq01             = _mm_mul_ps(iq0,jq1);
 154     qq02             = _mm_mul_ps(iq0,jq2);
 155     qq10             = _mm_mul_ps(iq1,jq0);
 156     qq11             = _mm_mul_ps(iq1,jq1);
 157     qq12             = _mm_mul_ps(iq1,jq2);
 158     qq20             = _mm_mul_ps(iq2,jq0);
 159     qq21             = _mm_mul_ps(iq2,jq1);
 160     qq22             = _mm_mul_ps(iq2,jq2);
 161
 162     /* Avoid stupid compiler warnings */
 163     jnrA = jnrB = jnrC = jnrD = 0;
 164     j_coord_offsetA = 0;
 165     j_coord_offsetB = 0;
 166     j_coord_offsetC = 0;
 167     j_coord_offsetD = 0;
 168
 169     outeriter        = 0;
 170     inneriter        = 0;
 171
 172     for(iidx=0;iidx<4*DIM;iidx++)
 173     {
 174         scratch[iidx] = 0.0;
 175     }
 176
 177     /* Start outer loop over neighborlists */
 178     for(iidx=0; iidx<nri; iidx++)
 179     {
 180         /* Load shift vector for this list */
 181         i_shift_offset   = DIM*shiftidx[iidx];
 182
 183         /* Load limits for loop over neighbors */
 184         j_index_start    = jindex[iidx];
 185         j_index_end      = jindex[iidx+1];
 186
 187         /* Get outer coordinate index */
 188         inr              = iinr[iidx];
 189         i_coord_offset   = DIM*inr;
 190
 191         /* Load i particle coords and add shift vector */
 192         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 193                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 194
 195         fix0             = _mm_setzero_ps();
 196         fiy0             = _mm_setzero_ps();
 197         fiz0             = _mm_setzero_ps();
 198         fix1             = _mm_setzero_ps();
 199         fiy1             = _mm_setzero_ps();
 200         fiz1             = _mm_setzero_ps();
 201         fix2             = _mm_setzero_ps();
 202         fiy2             = _mm_setzero_ps();
 203         fiz2             = _mm_setzero_ps();
 204
 205         /* Reset potential sums */
 206         velecsum         = _mm_setzero_ps();
 207         vvdwsum          = _mm_setzero_ps();
 208
 209         /* Start inner kernel loop */
 210         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 211         {
 212
 213             /* Get j neighbor index, and coordinate index */
 214             jnrA             = jjnr[jidx];
 215             jnrB             = jjnr[jidx+1];
 216             jnrC             = jjnr[jidx+2];
 217             jnrD             = jjnr[jidx+3];
 218             j_coord_offsetA  = DIM*jnrA;
 219             j_coord_offsetB  = DIM*jnrB;
 220             j_coord_offsetC  = DIM*jnrC;
 221             j_coord_offsetD  = DIM*jnrD;
 222
 223             /* load j atom coordinates */
 224             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 225                                               x+j_coord_offsetC,x+j_coord_offsetD,
 226                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 227
 228             /* Calculate displacement vector */
 229             dx00             = _mm_sub_ps(ix0,jx0);
 230             dy00             = _mm_sub_ps(iy0,jy0);
 231             dz00             = _mm_sub_ps(iz0,jz0);
 232             dx01             = _mm_sub_ps(ix0,jx1);
 233             dy01             = _mm_sub_ps(iy0,jy1);
 234             dz01             = _mm_sub_ps(iz0,jz1);
 235             dx02             = _mm_sub_ps(ix0,jx2);
 236             dy02             = _mm_sub_ps(iy0,jy2);
 237             dz02             = _mm_sub_ps(iz0,jz2);
 238             dx10             = _mm_sub_ps(ix1,jx0);
 239             dy10             = _mm_sub_ps(iy1,jy0);
 240             dz10             = _mm_sub_ps(iz1,jz0);
 241             dx11             = _mm_sub_ps(ix1,jx1);
 242             dy11             = _mm_sub_ps(iy1,jy1);
 243             dz11             = _mm_sub_ps(iz1,jz1);
 244             dx12             = _mm_sub_ps(ix1,jx2);
 245             dy12             = _mm_sub_ps(iy1,jy2);
 246             dz12             = _mm_sub_ps(iz1,jz2);
 247             dx20             = _mm_sub_ps(ix2,jx0);
 248             dy20             = _mm_sub_ps(iy2,jy0);
 249             dz20             = _mm_sub_ps(iz2,jz0);
 250             dx21             = _mm_sub_ps(ix2,jx1);
 251             dy21             = _mm_sub_ps(iy2,jy1);
 252             dz21             = _mm_sub_ps(iz2,jz1);
 253             dx22             = _mm_sub_ps(ix2,jx2);
 254             dy22             = _mm_sub_ps(iy2,jy2);
 255             dz22             = _mm_sub_ps(iz2,jz2);
 256
 257             /* Calculate squared distance and things based on it */
 258             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 259             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 260             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 261             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 262             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 263             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 264             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 265             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 266             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 267
 268             rinv00           = avx128fma_invsqrt_f(rsq00);
 269             rinv01           = avx128fma_invsqrt_f(rsq01);
 270             rinv02           = avx128fma_invsqrt_f(rsq02);
 271             rinv10           = avx128fma_invsqrt_f(rsq10);
 272             rinv11           = avx128fma_invsqrt_f(rsq11);
 273             rinv12           = avx128fma_invsqrt_f(rsq12);
 274             rinv20           = avx128fma_invsqrt_f(rsq20);
 275             rinv21           = avx128fma_invsqrt_f(rsq21);
 276             rinv22           = avx128fma_invsqrt_f(rsq22);
 277
 278             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 279
 280             fjx0             = _mm_setzero_ps();
 281             fjy0             = _mm_setzero_ps();
 282             fjz0             = _mm_setzero_ps();
 283             fjx1             = _mm_setzero_ps();
 284             fjy1             = _mm_setzero_ps();
 285             fjz1             = _mm_setzero_ps();
 286             fjx2             = _mm_setzero_ps();
 287             fjy2             = _mm_setzero_ps();
 288             fjz2             = _mm_setzero_ps();
 289
 290             /**************************
 291              * CALCULATE INTERACTIONS *
 292              **************************/
 293
 294             r00              = _mm_mul_ps(rsq00,rinv00);
 295
 296             /* Calculate table index by multiplying r with table scale and truncate to integer */
 297             rt               = _mm_mul_ps(r00,vftabscale);
 298             vfitab           = _mm_cvttps_epi32(rt);
 299 #ifdef __XOP__
 300             vfeps            = _mm_frcz_ps(rt);
 301 #else
 302             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 303 #endif
 304             twovfeps         = _mm_add_ps(vfeps,vfeps);
 305             vfitab           = _mm_slli_epi32(vfitab,2);
 306
 307             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 308             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 309             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 310             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 311             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 312             _MM_TRANSPOSE4_PS(Y,F,G,H);
 313             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 314             VV               = _mm_macc_ps(vfeps,Fp,Y);
 315             velec            = _mm_mul_ps(qq00,VV);
 316             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 317             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
 318
 319             /* LENNARD-JONES DISPERSION/REPULSION */
 320
 321             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 322             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 323             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 324             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 325             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 326
 327             /* Update potential sum for this i atom from the interaction with this j atom. */
 328             velecsum         = _mm_add_ps(velecsum,velec);
 329             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 330
 331             fscal            = _mm_add_ps(felec,fvdw);
 332
 333              /* Update vectorial force */
 334             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 335             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 336             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 337
 338             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 339             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 340             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 341
 342             /**************************
 343              * CALCULATE INTERACTIONS *
 344              **************************/
 345
 346             r01              = _mm_mul_ps(rsq01,rinv01);
 347
 348             /* Calculate table index by multiplying r with table scale and truncate to integer */
 349             rt               = _mm_mul_ps(r01,vftabscale);
 350             vfitab           = _mm_cvttps_epi32(rt);
 351 #ifdef __XOP__
 352             vfeps            = _mm_frcz_ps(rt);
 353 #else
 354             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 355 #endif
 356             twovfeps         = _mm_add_ps(vfeps,vfeps);
 357             vfitab           = _mm_slli_epi32(vfitab,2);
 358
 359             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 360             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 361             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 362             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 363             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 364             _MM_TRANSPOSE4_PS(Y,F,G,H);
 365             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 366             VV               = _mm_macc_ps(vfeps,Fp,Y);
 367             velec            = _mm_mul_ps(qq01,VV);
 368             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 369             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
 370
 371             /* Update potential sum for this i atom from the interaction with this j atom. */
 372             velecsum         = _mm_add_ps(velecsum,velec);
 373
 374             fscal            = felec;
 375
 376              /* Update vectorial force */
 377             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 378             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 379             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 380
 381             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 382             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 383             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 384
 385             /**************************
 386              * CALCULATE INTERACTIONS *
 387              **************************/
 388
 389             r02              = _mm_mul_ps(rsq02,rinv02);
 390
 391             /* Calculate table index by multiplying r with table scale and truncate to integer */
 392             rt               = _mm_mul_ps(r02,vftabscale);
 393             vfitab           = _mm_cvttps_epi32(rt);
 394 #ifdef __XOP__
 395             vfeps            = _mm_frcz_ps(rt);
 396 #else
 397             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 398 #endif
 399             twovfeps         = _mm_add_ps(vfeps,vfeps);
 400             vfitab           = _mm_slli_epi32(vfitab,2);
 401
 402             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 403             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 404             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 405             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 406             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 407             _MM_TRANSPOSE4_PS(Y,F,G,H);
 408             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 409             VV               = _mm_macc_ps(vfeps,Fp,Y);
 410             velec            = _mm_mul_ps(qq02,VV);
 411             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 412             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
 413
 414             /* Update potential sum for this i atom from the interaction with this j atom. */
 415             velecsum         = _mm_add_ps(velecsum,velec);
 416
 417             fscal            = felec;
 418
 419              /* Update vectorial force */
 420             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 421             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 422             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 423
 424             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 425             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 426             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 427
 428             /**************************
 429              * CALCULATE INTERACTIONS *
 430              **************************/
 431
 432             r10              = _mm_mul_ps(rsq10,rinv10);
 433
 434             /* Calculate table index by multiplying r with table scale and truncate to integer */
 435             rt               = _mm_mul_ps(r10,vftabscale);
 436             vfitab           = _mm_cvttps_epi32(rt);
 437 #ifdef __XOP__
 438             vfeps            = _mm_frcz_ps(rt);
 439 #else
 440             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 441 #endif
 442             twovfeps         = _mm_add_ps(vfeps,vfeps);
 443             vfitab           = _mm_slli_epi32(vfitab,2);
 444
 445             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 446             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 447             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 448             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 449             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 450             _MM_TRANSPOSE4_PS(Y,F,G,H);
 451             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 452             VV               = _mm_macc_ps(vfeps,Fp,Y);
 453             velec            = _mm_mul_ps(qq10,VV);
 454             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 455             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
 456
 457             /* Update potential sum for this i atom from the interaction with this j atom. */
 458             velecsum         = _mm_add_ps(velecsum,velec);
 459
 460             fscal            = felec;
 461
 462              /* Update vectorial force */
 463             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 464             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 465             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 466
 467             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 468             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 469             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 470
 471             /**************************
 472              * CALCULATE INTERACTIONS *
 473              **************************/
 474
 475             r11              = _mm_mul_ps(rsq11,rinv11);
 476
 477             /* Calculate table index by multiplying r with table scale and truncate to integer */
 478             rt               = _mm_mul_ps(r11,vftabscale);
 479             vfitab           = _mm_cvttps_epi32(rt);
 480 #ifdef __XOP__
 481             vfeps            = _mm_frcz_ps(rt);
 482 #else
 483             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 484 #endif
 485             twovfeps         = _mm_add_ps(vfeps,vfeps);
 486             vfitab           = _mm_slli_epi32(vfitab,2);
 487
 488             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 489             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 490             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 491             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 492             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 493             _MM_TRANSPOSE4_PS(Y,F,G,H);
 494             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 495             VV               = _mm_macc_ps(vfeps,Fp,Y);
 496             velec            = _mm_mul_ps(qq11,VV);
 497             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 498             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
 499
 500             /* Update potential sum for this i atom from the interaction with this j atom. */
 501             velecsum         = _mm_add_ps(velecsum,velec);
 502
 503             fscal            = felec;
 504
 505              /* Update vectorial force */
 506             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 507             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 508             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 509
 510             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 511             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 512             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 513
 514             /**************************
 515              * CALCULATE INTERACTIONS *
 516              **************************/
 517
 518             r12              = _mm_mul_ps(rsq12,rinv12);
 519
 520             /* Calculate table index by multiplying r with table scale and truncate to integer */
 521             rt               = _mm_mul_ps(r12,vftabscale);
 522             vfitab           = _mm_cvttps_epi32(rt);
 523 #ifdef __XOP__
 524             vfeps            = _mm_frcz_ps(rt);
 525 #else
 526             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 527 #endif
 528             twovfeps         = _mm_add_ps(vfeps,vfeps);
 529             vfitab           = _mm_slli_epi32(vfitab,2);
 530
 531             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 532             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 533             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 534             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 535             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 536             _MM_TRANSPOSE4_PS(Y,F,G,H);
 537             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 538             VV               = _mm_macc_ps(vfeps,Fp,Y);
 539             velec            = _mm_mul_ps(qq12,VV);
 540             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 541             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
 542
 543             /* Update potential sum for this i atom from the interaction with this j atom. */
 544             velecsum         = _mm_add_ps(velecsum,velec);
 545
 546             fscal            = felec;
 547
 548              /* Update vectorial force */
 549             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 550             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 551             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 552
 553             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 554             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 555             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 556
 557             /**************************
 558              * CALCULATE INTERACTIONS *
 559              **************************/
 560
 561             r20              = _mm_mul_ps(rsq20,rinv20);
 562
 563             /* Calculate table index by multiplying r with table scale and truncate to integer */
 564             rt               = _mm_mul_ps(r20,vftabscale);
 565             vfitab           = _mm_cvttps_epi32(rt);
 566 #ifdef __XOP__
 567             vfeps            = _mm_frcz_ps(rt);
 568 #else
 569             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 570 #endif
 571             twovfeps         = _mm_add_ps(vfeps,vfeps);
 572             vfitab           = _mm_slli_epi32(vfitab,2);
 573
 574             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 575             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 576             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 577             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 578             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 579             _MM_TRANSPOSE4_PS(Y,F,G,H);
 580             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 581             VV               = _mm_macc_ps(vfeps,Fp,Y);
 582             velec            = _mm_mul_ps(qq20,VV);
 583             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 584             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
 585
 586             /* Update potential sum for this i atom from the interaction with this j atom. */
 587             velecsum         = _mm_add_ps(velecsum,velec);
 588
 589             fscal            = felec;
 590
 591              /* Update vectorial force */
 592             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 593             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 594             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 595
 596             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 597             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 598             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 599
 600             /**************************
 601              * CALCULATE INTERACTIONS *
 602              **************************/
 603
 604             r21              = _mm_mul_ps(rsq21,rinv21);
 605
 606             /* Calculate table index by multiplying r with table scale and truncate to integer */
 607             rt               = _mm_mul_ps(r21,vftabscale);
 608             vfitab           = _mm_cvttps_epi32(rt);
 609 #ifdef __XOP__
 610             vfeps            = _mm_frcz_ps(rt);
 611 #else
 612             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 613 #endif
 614             twovfeps         = _mm_add_ps(vfeps,vfeps);
 615             vfitab           = _mm_slli_epi32(vfitab,2);
 616
 617             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 618             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 619             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 620             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 621             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 622             _MM_TRANSPOSE4_PS(Y,F,G,H);
 623             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 624             VV               = _mm_macc_ps(vfeps,Fp,Y);
 625             velec            = _mm_mul_ps(qq21,VV);
 626             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 627             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
 628
 629             /* Update potential sum for this i atom from the interaction with this j atom. */
 630             velecsum         = _mm_add_ps(velecsum,velec);
 631
 632             fscal            = felec;
 633
 634              /* Update vectorial force */
 635             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 636             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 637             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 638
 639             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 640             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 641             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 642
 643             /**************************
 644              * CALCULATE INTERACTIONS *
 645              **************************/
 646
 647             r22              = _mm_mul_ps(rsq22,rinv22);
 648
 649             /* Calculate table index by multiplying r with table scale and truncate to integer */
 650             rt               = _mm_mul_ps(r22,vftabscale);
 651             vfitab           = _mm_cvttps_epi32(rt);
 652 #ifdef __XOP__
 653             vfeps            = _mm_frcz_ps(rt);
 654 #else
 655             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 656 #endif
 657             twovfeps         = _mm_add_ps(vfeps,vfeps);
 658             vfitab           = _mm_slli_epi32(vfitab,2);
 659
 660             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 661             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 662             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 663             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 664             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 665             _MM_TRANSPOSE4_PS(Y,F,G,H);
 666             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 667             VV               = _mm_macc_ps(vfeps,Fp,Y);
 668             velec            = _mm_mul_ps(qq22,VV);
 669             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 670             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
 671
 672             /* Update potential sum for this i atom from the interaction with this j atom. */
 673             velecsum         = _mm_add_ps(velecsum,velec);
 674
 675             fscal            = felec;
 676
 677              /* Update vectorial force */
 678             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 679             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 680             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 681
 682             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 683             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 684             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 685
 686             fjptrA             = f+j_coord_offsetA;
 687             fjptrB             = f+j_coord_offsetB;
 688             fjptrC             = f+j_coord_offsetC;
 689             fjptrD             = f+j_coord_offsetD;
 690
 691             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 692                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 693
 694             /* Inner loop uses 427 flops */
 695         }
 696
 697         if(jidx<j_index_end)
 698         {
 699
 700             /* Get j neighbor index, and coordinate index */
 701             jnrlistA         = jjnr[jidx];
 702             jnrlistB         = jjnr[jidx+1];
 703             jnrlistC         = jjnr[jidx+2];
 704             jnrlistD         = jjnr[jidx+3];
 705             /* Sign of each element will be negative for non-real atoms.
 706              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 707              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 708              */
 709             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 710             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 711             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 712             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 713             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 714             j_coord_offsetA  = DIM*jnrA;
 715             j_coord_offsetB  = DIM*jnrB;
 716             j_coord_offsetC  = DIM*jnrC;
 717             j_coord_offsetD  = DIM*jnrD;
 718
 719             /* load j atom coordinates */
 720             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 721                                               x+j_coord_offsetC,x+j_coord_offsetD,
 722                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 723
 724             /* Calculate displacement vector */
 725             dx00             = _mm_sub_ps(ix0,jx0);
 726             dy00             = _mm_sub_ps(iy0,jy0);
 727             dz00             = _mm_sub_ps(iz0,jz0);
 728             dx01             = _mm_sub_ps(ix0,jx1);
 729             dy01             = _mm_sub_ps(iy0,jy1);
 730             dz01             = _mm_sub_ps(iz0,jz1);
 731             dx02             = _mm_sub_ps(ix0,jx2);
 732             dy02             = _mm_sub_ps(iy0,jy2);
 733             dz02             = _mm_sub_ps(iz0,jz2);
 734             dx10             = _mm_sub_ps(ix1,jx0);
 735             dy10             = _mm_sub_ps(iy1,jy0);
 736             dz10             = _mm_sub_ps(iz1,jz0);
 737             dx11             = _mm_sub_ps(ix1,jx1);
 738             dy11             = _mm_sub_ps(iy1,jy1);
 739             dz11             = _mm_sub_ps(iz1,jz1);
 740             dx12             = _mm_sub_ps(ix1,jx2);
 741             dy12             = _mm_sub_ps(iy1,jy2);
 742             dz12             = _mm_sub_ps(iz1,jz2);
 743             dx20             = _mm_sub_ps(ix2,jx0);
 744             dy20             = _mm_sub_ps(iy2,jy0);
 745             dz20             = _mm_sub_ps(iz2,jz0);
 746             dx21             = _mm_sub_ps(ix2,jx1);
 747             dy21             = _mm_sub_ps(iy2,jy1);
 748             dz21             = _mm_sub_ps(iz2,jz1);
 749             dx22             = _mm_sub_ps(ix2,jx2);
 750             dy22             = _mm_sub_ps(iy2,jy2);
 751             dz22             = _mm_sub_ps(iz2,jz2);
 752
 753             /* Calculate squared distance and things based on it */
 754             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 755             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 756             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 757             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 758             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 759             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 760             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 761             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 762             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 763
 764             rinv00           = avx128fma_invsqrt_f(rsq00);
 765             rinv01           = avx128fma_invsqrt_f(rsq01);
 766             rinv02           = avx128fma_invsqrt_f(rsq02);
 767             rinv10           = avx128fma_invsqrt_f(rsq10);
 768             rinv11           = avx128fma_invsqrt_f(rsq11);
 769             rinv12           = avx128fma_invsqrt_f(rsq12);
 770             rinv20           = avx128fma_invsqrt_f(rsq20);
 771             rinv21           = avx128fma_invsqrt_f(rsq21);
 772             rinv22           = avx128fma_invsqrt_f(rsq22);
 773
 774             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 775
 776             fjx0             = _mm_setzero_ps();
 777             fjy0             = _mm_setzero_ps();
 778             fjz0             = _mm_setzero_ps();
 779             fjx1             = _mm_setzero_ps();
 780             fjy1             = _mm_setzero_ps();
 781             fjz1             = _mm_setzero_ps();
 782             fjx2             = _mm_setzero_ps();
 783             fjy2             = _mm_setzero_ps();
 784             fjz2             = _mm_setzero_ps();
 785
 786             /**************************
 787              * CALCULATE INTERACTIONS *
 788              **************************/
 789
 790             r00              = _mm_mul_ps(rsq00,rinv00);
 791             r00              = _mm_andnot_ps(dummy_mask,r00);
 792
 793             /* Calculate table index by multiplying r with table scale and truncate to integer */
 794             rt               = _mm_mul_ps(r00,vftabscale);
 795             vfitab           = _mm_cvttps_epi32(rt);
 796 #ifdef __XOP__
 797             vfeps            = _mm_frcz_ps(rt);
 798 #else
 799             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 800 #endif
 801             twovfeps         = _mm_add_ps(vfeps,vfeps);
 802             vfitab           = _mm_slli_epi32(vfitab,2);
 803
 804             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 805             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 806             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 807             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 808             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 809             _MM_TRANSPOSE4_PS(Y,F,G,H);
 810             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 811             VV               = _mm_macc_ps(vfeps,Fp,Y);
 812             velec            = _mm_mul_ps(qq00,VV);
 813             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 814             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
 815
 816             /* LENNARD-JONES DISPERSION/REPULSION */
 817
 818             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 819             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 820             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 821             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 822             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 823
 824             /* Update potential sum for this i atom from the interaction with this j atom. */
 825             velec            = _mm_andnot_ps(dummy_mask,velec);
 826             velecsum         = _mm_add_ps(velecsum,velec);
 827             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 828             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 829
 830             fscal            = _mm_add_ps(felec,fvdw);
 831
 832             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 833
 834              /* Update vectorial force */
 835             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 836             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 837             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 838
 839             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 840             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 841             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 842
 843             /**************************
 844              * CALCULATE INTERACTIONS *
 845              **************************/
 846
 847             r01              = _mm_mul_ps(rsq01,rinv01);
 848             r01              = _mm_andnot_ps(dummy_mask,r01);
 849
 850             /* Calculate table index by multiplying r with table scale and truncate to integer */
 851             rt               = _mm_mul_ps(r01,vftabscale);
 852             vfitab           = _mm_cvttps_epi32(rt);
 853 #ifdef __XOP__
 854             vfeps            = _mm_frcz_ps(rt);
 855 #else
 856             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 857 #endif
 858             twovfeps         = _mm_add_ps(vfeps,vfeps);
 859             vfitab           = _mm_slli_epi32(vfitab,2);
 860
 861             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 862             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 863             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 864             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 865             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 866             _MM_TRANSPOSE4_PS(Y,F,G,H);
 867             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 868             VV               = _mm_macc_ps(vfeps,Fp,Y);
 869             velec            = _mm_mul_ps(qq01,VV);
 870             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 871             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
 872
 873             /* Update potential sum for this i atom from the interaction with this j atom. */
 874             velec            = _mm_andnot_ps(dummy_mask,velec);
 875             velecsum         = _mm_add_ps(velecsum,velec);
 876
 877             fscal            = felec;
 878
 879             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 880
 881              /* Update vectorial force */
 882             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 883             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 884             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 885
 886             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 887             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 888             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 889
 890             /**************************
 891              * CALCULATE INTERACTIONS *
 892              **************************/
 893
 894             r02              = _mm_mul_ps(rsq02,rinv02);
 895             r02              = _mm_andnot_ps(dummy_mask,r02);
 896
 897             /* Calculate table index by multiplying r with table scale and truncate to integer */
 898             rt               = _mm_mul_ps(r02,vftabscale);
 899             vfitab           = _mm_cvttps_epi32(rt);
 900 #ifdef __XOP__
 901             vfeps            = _mm_frcz_ps(rt);
 902 #else
 903             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 904 #endif
 905             twovfeps         = _mm_add_ps(vfeps,vfeps);
 906             vfitab           = _mm_slli_epi32(vfitab,2);
 907
 908             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 909             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 910             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 911             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 912             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 913             _MM_TRANSPOSE4_PS(Y,F,G,H);
 914             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 915             VV               = _mm_macc_ps(vfeps,Fp,Y);
 916             velec            = _mm_mul_ps(qq02,VV);
 917             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 918             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
 919
 920             /* Update potential sum for this i atom from the interaction with this j atom. */
 921             velec            = _mm_andnot_ps(dummy_mask,velec);
 922             velecsum         = _mm_add_ps(velecsum,velec);
 923
 924             fscal            = felec;
 925
 926             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 927
 928              /* Update vectorial force */
 929             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 930             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 931             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 932
 933             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 934             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 935             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 936
 937             /**************************
 938              * CALCULATE INTERACTIONS *
 939              **************************/
 940
 941             r10              = _mm_mul_ps(rsq10,rinv10);
 942             r10              = _mm_andnot_ps(dummy_mask,r10);
 943
 944             /* Calculate table index by multiplying r with table scale and truncate to integer */
 945             rt               = _mm_mul_ps(r10,vftabscale);
 946             vfitab           = _mm_cvttps_epi32(rt);
 947 #ifdef __XOP__
 948             vfeps            = _mm_frcz_ps(rt);
 949 #else
 950             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 951 #endif
 952             twovfeps         = _mm_add_ps(vfeps,vfeps);
 953             vfitab           = _mm_slli_epi32(vfitab,2);
 954
 955             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 956             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
 957             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
 958             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
 959             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
 960             _MM_TRANSPOSE4_PS(Y,F,G,H);
 961             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
 962             VV               = _mm_macc_ps(vfeps,Fp,Y);
 963             velec            = _mm_mul_ps(qq10,VV);
 964             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
 965             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
 966
 967             /* Update potential sum for this i atom from the interaction with this j atom. */
 968             velec            = _mm_andnot_ps(dummy_mask,velec);
 969             velecsum         = _mm_add_ps(velecsum,velec);
 970
 971             fscal            = felec;
 972
 973             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 974
 975              /* Update vectorial force */
 976             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 977             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 978             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 979
 980             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 981             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 982             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 983
 984             /**************************
 985              * CALCULATE INTERACTIONS *
 986              **************************/
 987
 988             r11              = _mm_mul_ps(rsq11,rinv11);
 989             r11              = _mm_andnot_ps(dummy_mask,r11);
 990
 991             /* Calculate table index by multiplying r with table scale and truncate to integer */
 992             rt               = _mm_mul_ps(r11,vftabscale);
 993             vfitab           = _mm_cvttps_epi32(rt);
 994 #ifdef __XOP__
 995             vfeps            = _mm_frcz_ps(rt);
 996 #else
 997             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
 998 #endif
 999             twovfeps         = _mm_add_ps(vfeps,vfeps);
1000             vfitab           = _mm_slli_epi32(vfitab,2);
1001
1002             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1003             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1004             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1005             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1006             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1007             _MM_TRANSPOSE4_PS(Y,F,G,H);
1008             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1009             VV               = _mm_macc_ps(vfeps,Fp,Y);
1010             velec            = _mm_mul_ps(qq11,VV);
1011             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1012             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
1013
1014             /* Update potential sum for this i atom from the interaction with this j atom. */
1015             velec            = _mm_andnot_ps(dummy_mask,velec);
1016             velecsum         = _mm_add_ps(velecsum,velec);
1017
1018             fscal            = felec;
1019
1020             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1021
1022              /* Update vectorial force */
1023             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1024             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1025             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1026
1027             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1028             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1029             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1030
1031             /**************************
1032              * CALCULATE INTERACTIONS *
1033              **************************/
1034
1035             r12              = _mm_mul_ps(rsq12,rinv12);
1036             r12              = _mm_andnot_ps(dummy_mask,r12);
1037
1038             /* Calculate table index by multiplying r with table scale and truncate to integer */
1039             rt               = _mm_mul_ps(r12,vftabscale);
1040             vfitab           = _mm_cvttps_epi32(rt);
1041 #ifdef __XOP__
1042             vfeps            = _mm_frcz_ps(rt);
1043 #else
1044             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1045 #endif
1046             twovfeps         = _mm_add_ps(vfeps,vfeps);
1047             vfitab           = _mm_slli_epi32(vfitab,2);
1048
1049             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1050             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1051             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1052             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1053             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1054             _MM_TRANSPOSE4_PS(Y,F,G,H);
1055             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1056             VV               = _mm_macc_ps(vfeps,Fp,Y);
1057             velec            = _mm_mul_ps(qq12,VV);
1058             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1059             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
1060
1061             /* Update potential sum for this i atom from the interaction with this j atom. */
1062             velec            = _mm_andnot_ps(dummy_mask,velec);
1063             velecsum         = _mm_add_ps(velecsum,velec);
1064
1065             fscal            = felec;
1066
1067             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1068
1069              /* Update vectorial force */
1070             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1071             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1072             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1073
1074             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1075             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1076             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1077
1078             /**************************
1079              * CALCULATE INTERACTIONS *
1080              **************************/
1081
1082             r20              = _mm_mul_ps(rsq20,rinv20);
1083             r20              = _mm_andnot_ps(dummy_mask,r20);
1084
1085             /* Calculate table index by multiplying r with table scale and truncate to integer */
1086             rt               = _mm_mul_ps(r20,vftabscale);
1087             vfitab           = _mm_cvttps_epi32(rt);
1088 #ifdef __XOP__
1089             vfeps            = _mm_frcz_ps(rt);
1090 #else
1091             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1092 #endif
1093             twovfeps         = _mm_add_ps(vfeps,vfeps);
1094             vfitab           = _mm_slli_epi32(vfitab,2);
1095
1096             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1097             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1098             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1099             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1100             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1101             _MM_TRANSPOSE4_PS(Y,F,G,H);
1102             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1103             VV               = _mm_macc_ps(vfeps,Fp,Y);
1104             velec            = _mm_mul_ps(qq20,VV);
1105             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1106             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1107
1108             /* Update potential sum for this i atom from the interaction with this j atom. */
1109             velec            = _mm_andnot_ps(dummy_mask,velec);
1110             velecsum         = _mm_add_ps(velecsum,velec);
1111
1112             fscal            = felec;
1113
1114             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1115
1116              /* Update vectorial force */
1117             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1118             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1119             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1120
1121             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1122             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1123             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1124
1125             /**************************
1126              * CALCULATE INTERACTIONS *
1127              **************************/
1128
1129             r21              = _mm_mul_ps(rsq21,rinv21);
1130             r21              = _mm_andnot_ps(dummy_mask,r21);
1131
1132             /* Calculate table index by multiplying r with table scale and truncate to integer */
1133             rt               = _mm_mul_ps(r21,vftabscale);
1134             vfitab           = _mm_cvttps_epi32(rt);
1135 #ifdef __XOP__
1136             vfeps            = _mm_frcz_ps(rt);
1137 #else
1138             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1139 #endif
1140             twovfeps         = _mm_add_ps(vfeps,vfeps);
1141             vfitab           = _mm_slli_epi32(vfitab,2);
1142
1143             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1144             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1145             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1146             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1147             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1148             _MM_TRANSPOSE4_PS(Y,F,G,H);
1149             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1150             VV               = _mm_macc_ps(vfeps,Fp,Y);
1151             velec            = _mm_mul_ps(qq21,VV);
1152             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1153             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
1154
1155             /* Update potential sum for this i atom from the interaction with this j atom. */
1156             velec            = _mm_andnot_ps(dummy_mask,velec);
1157             velecsum         = _mm_add_ps(velecsum,velec);
1158
1159             fscal            = felec;
1160
1161             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1162
1163              /* Update vectorial force */
1164             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1165             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1166             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1167
1168             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1169             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1170             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1171
1172             /**************************
1173              * CALCULATE INTERACTIONS *
1174              **************************/
1175
1176             r22              = _mm_mul_ps(rsq22,rinv22);
1177             r22              = _mm_andnot_ps(dummy_mask,r22);
1178
1179             /* Calculate table index by multiplying r with table scale and truncate to integer */
1180             rt               = _mm_mul_ps(r22,vftabscale);
1181             vfitab           = _mm_cvttps_epi32(rt);
1182 #ifdef __XOP__
1183             vfeps            = _mm_frcz_ps(rt);
1184 #else
1185             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1186 #endif
1187             twovfeps         = _mm_add_ps(vfeps,vfeps);
1188             vfitab           = _mm_slli_epi32(vfitab,2);
1189
1190             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1191             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1192             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1193             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1194             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1195             _MM_TRANSPOSE4_PS(Y,F,G,H);
1196             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1197             VV               = _mm_macc_ps(vfeps,Fp,Y);
1198             velec            = _mm_mul_ps(qq22,VV);
1199             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1200             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
1201
1202             /* Update potential sum for this i atom from the interaction with this j atom. */
1203             velec            = _mm_andnot_ps(dummy_mask,velec);
1204             velecsum         = _mm_add_ps(velecsum,velec);
1205
1206             fscal            = felec;
1207
1208             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1209
1210              /* Update vectorial force */
1211             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1212             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1213             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1214
1215             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1216             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1217             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1218
1219             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1220             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1221             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1222             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1223
1224             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1225                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1226
1227             /* Inner loop uses 436 flops */
1228         }
1229
1230         /* End of innermost loop */
1231
1232         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1233                                               f+i_coord_offset,fshift+i_shift_offset);
1234
1235         ggid                        = gid[iidx];
1236         /* Update potential energies */
1237         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1238         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1239
1240         /* Increment number of inner iterations */
1241         inneriter                  += j_index_end - j_index_start;
1242
1243         /* Outer loop uses 20 flops */
1244     }
1245
1246     /* Increment number of outer iterations */
1247     outeriter        += nri;
1248
1249     /* Update outer/inner flops */
1250
1251     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*436);
1252 }
1253 /*
1254  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_F_avx_128_fma_single
1255  * Electrostatics interaction: CubicSplineTable
1256  * VdW interaction:            LennardJones
1257  * Geometry:                   Water3-Water3
1258  * Calculate force/pot:        Force
1259  */
1260 void
1261 nb_kernel_ElecCSTab_VdwLJ_GeomW3W3_F_avx_128_fma_single
1262                     (t_nblist                    * gmx_restrict       nlist,
1263                      rvec                        * gmx_restrict          xx,
1264                      rvec                        * gmx_restrict          ff,
1265                      struct t_forcerec           * gmx_restrict          fr,
1266                      t_mdatoms                   * gmx_restrict     mdatoms,
1267                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1268                      t_nrnb                      * gmx_restrict        nrnb)
1269 {
1270     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1271      * just 0 for non-waters.
1272      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1273      * jnr indices corresponding to data put in the four positions in the SIMD register.
1274      */
1275     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
1276     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1277     int              jnrA,jnrB,jnrC,jnrD;
1278     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1279     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1280     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
1281     real             rcutoff_scalar;
1282     real             *shiftvec,*fshift,*x,*f;
1283     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1284     real             scratch[4*DIM];
1285     __m128           fscal,rcutoff,rcutoff2,jidxall;
1286     int              vdwioffset0;
1287     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1288     int              vdwioffset1;
1289     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1290     int              vdwioffset2;
1291     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1292     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1293     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1294     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1295     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1296     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1297     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1298     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1299     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1300     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1301     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1302     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1303     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1304     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1305     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1306     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1307     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
1308     real             *charge;
1309     int              nvdwtype;
1310     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1311     int              *vdwtype;
1312     real             *vdwparam;
1313     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
1314     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
1315     __m128i          vfitab;
1316     __m128i          ifour       = _mm_set1_epi32(4);
1317     __m128           rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
1318     real             *vftab;
1319     __m128           dummy_mask,cutoff_mask;
1320     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1321     __m128           one     = _mm_set1_ps(1.0);
1322     __m128           two     = _mm_set1_ps(2.0);
1323     x                = xx[0];
1324     f                = ff[0];
1325
1326     nri              = nlist->nri;
1327     iinr             = nlist->iinr;
1328     jindex           = nlist->jindex;
1329     jjnr             = nlist->jjnr;
1330     shiftidx         = nlist->shift;
1331     gid              = nlist->gid;
1332     shiftvec         = fr->shift_vec[0];
1333     fshift           = fr->fshift[0];
1334     facel            = _mm_set1_ps(fr->ic->epsfac);
1335     charge           = mdatoms->chargeA;
1336     nvdwtype         = fr->ntype;
1337     vdwparam         = fr->nbfp;
1338     vdwtype          = mdatoms->typeA;
1339
1340     vftab            = kernel_data->table_elec->data;
1341     vftabscale       = _mm_set1_ps(kernel_data->table_elec->scale);
1342
1343     /* Setup water-specific parameters */
1344     inr              = nlist->iinr[0];
1345     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1346     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1347     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1348     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
1349
1350     jq0              = _mm_set1_ps(charge[inr+0]);
1351     jq1              = _mm_set1_ps(charge[inr+1]);
1352     jq2              = _mm_set1_ps(charge[inr+2]);
1353     vdwjidx0A        = 2*vdwtype[inr+0];
1354     qq00             = _mm_mul_ps(iq0,jq0);
1355     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1356     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1357     qq01             = _mm_mul_ps(iq0,jq1);
1358     qq02             = _mm_mul_ps(iq0,jq2);
1359     qq10             = _mm_mul_ps(iq1,jq0);
1360     qq11             = _mm_mul_ps(iq1,jq1);
1361     qq12             = _mm_mul_ps(iq1,jq2);
1362     qq20             = _mm_mul_ps(iq2,jq0);
1363     qq21             = _mm_mul_ps(iq2,jq1);
1364     qq22             = _mm_mul_ps(iq2,jq2);
1365
1366     /* Avoid stupid compiler warnings */
1367     jnrA = jnrB = jnrC = jnrD = 0;
1368     j_coord_offsetA = 0;
1369     j_coord_offsetB = 0;
1370     j_coord_offsetC = 0;
1371     j_coord_offsetD = 0;
1372
1373     outeriter        = 0;
1374     inneriter        = 0;
1375
1376     for(iidx=0;iidx<4*DIM;iidx++)
1377     {
1378         scratch[iidx] = 0.0;
1379     }
1380
1381     /* Start outer loop over neighborlists */
1382     for(iidx=0; iidx<nri; iidx++)
1383     {
1384         /* Load shift vector for this list */
1385         i_shift_offset   = DIM*shiftidx[iidx];
1386
1387         /* Load limits for loop over neighbors */
1388         j_index_start    = jindex[iidx];
1389         j_index_end      = jindex[iidx+1];
1390
1391         /* Get outer coordinate index */
1392         inr              = iinr[iidx];
1393         i_coord_offset   = DIM*inr;
1394
1395         /* Load i particle coords and add shift vector */
1396         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1397                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1398
1399         fix0             = _mm_setzero_ps();
1400         fiy0             = _mm_setzero_ps();
1401         fiz0             = _mm_setzero_ps();
1402         fix1             = _mm_setzero_ps();
1403         fiy1             = _mm_setzero_ps();
1404         fiz1             = _mm_setzero_ps();
1405         fix2             = _mm_setzero_ps();
1406         fiy2             = _mm_setzero_ps();
1407         fiz2             = _mm_setzero_ps();
1408
1409         /* Start inner kernel loop */
1410         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1411         {
1412
1413             /* Get j neighbor index, and coordinate index */
1414             jnrA             = jjnr[jidx];
1415             jnrB             = jjnr[jidx+1];
1416             jnrC             = jjnr[jidx+2];
1417             jnrD             = jjnr[jidx+3];
1418             j_coord_offsetA  = DIM*jnrA;
1419             j_coord_offsetB  = DIM*jnrB;
1420             j_coord_offsetC  = DIM*jnrC;
1421             j_coord_offsetD  = DIM*jnrD;
1422
1423             /* load j atom coordinates */
1424             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1425                                               x+j_coord_offsetC,x+j_coord_offsetD,
1426                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1427
1428             /* Calculate displacement vector */
1429             dx00             = _mm_sub_ps(ix0,jx0);
1430             dy00             = _mm_sub_ps(iy0,jy0);
1431             dz00             = _mm_sub_ps(iz0,jz0);
1432             dx01             = _mm_sub_ps(ix0,jx1);
1433             dy01             = _mm_sub_ps(iy0,jy1);
1434             dz01             = _mm_sub_ps(iz0,jz1);
1435             dx02             = _mm_sub_ps(ix0,jx2);
1436             dy02             = _mm_sub_ps(iy0,jy2);
1437             dz02             = _mm_sub_ps(iz0,jz2);
1438             dx10             = _mm_sub_ps(ix1,jx0);
1439             dy10             = _mm_sub_ps(iy1,jy0);
1440             dz10             = _mm_sub_ps(iz1,jz0);
1441             dx11             = _mm_sub_ps(ix1,jx1);
1442             dy11             = _mm_sub_ps(iy1,jy1);
1443             dz11             = _mm_sub_ps(iz1,jz1);
1444             dx12             = _mm_sub_ps(ix1,jx2);
1445             dy12             = _mm_sub_ps(iy1,jy2);
1446             dz12             = _mm_sub_ps(iz1,jz2);
1447             dx20             = _mm_sub_ps(ix2,jx0);
1448             dy20             = _mm_sub_ps(iy2,jy0);
1449             dz20             = _mm_sub_ps(iz2,jz0);
1450             dx21             = _mm_sub_ps(ix2,jx1);
1451             dy21             = _mm_sub_ps(iy2,jy1);
1452             dz21             = _mm_sub_ps(iz2,jz1);
1453             dx22             = _mm_sub_ps(ix2,jx2);
1454             dy22             = _mm_sub_ps(iy2,jy2);
1455             dz22             = _mm_sub_ps(iz2,jz2);
1456
1457             /* Calculate squared distance and things based on it */
1458             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1459             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1460             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1461             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1462             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1463             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1464             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1465             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1466             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1467
1468             rinv00           = avx128fma_invsqrt_f(rsq00);
1469             rinv01           = avx128fma_invsqrt_f(rsq01);
1470             rinv02           = avx128fma_invsqrt_f(rsq02);
1471             rinv10           = avx128fma_invsqrt_f(rsq10);
1472             rinv11           = avx128fma_invsqrt_f(rsq11);
1473             rinv12           = avx128fma_invsqrt_f(rsq12);
1474             rinv20           = avx128fma_invsqrt_f(rsq20);
1475             rinv21           = avx128fma_invsqrt_f(rsq21);
1476             rinv22           = avx128fma_invsqrt_f(rsq22);
1477
1478             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1479
1480             fjx0             = _mm_setzero_ps();
1481             fjy0             = _mm_setzero_ps();
1482             fjz0             = _mm_setzero_ps();
1483             fjx1             = _mm_setzero_ps();
1484             fjy1             = _mm_setzero_ps();
1485             fjz1             = _mm_setzero_ps();
1486             fjx2             = _mm_setzero_ps();
1487             fjy2             = _mm_setzero_ps();
1488             fjz2             = _mm_setzero_ps();
1489
1490             /**************************
1491              * CALCULATE INTERACTIONS *
1492              **************************/
1493
1494             r00              = _mm_mul_ps(rsq00,rinv00);
1495
1496             /* Calculate table index by multiplying r with table scale and truncate to integer */
1497             rt               = _mm_mul_ps(r00,vftabscale);
1498             vfitab           = _mm_cvttps_epi32(rt);
1499 #ifdef __XOP__
1500             vfeps            = _mm_frcz_ps(rt);
1501 #else
1502             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1503 #endif
1504             twovfeps         = _mm_add_ps(vfeps,vfeps);
1505             vfitab           = _mm_slli_epi32(vfitab,2);
1506
1507             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1508             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1509             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1510             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1511             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1512             _MM_TRANSPOSE4_PS(Y,F,G,H);
1513             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1514             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1515             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
1516
1517             /* LENNARD-JONES DISPERSION/REPULSION */
1518
1519             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1520             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1521
1522             fscal            = _mm_add_ps(felec,fvdw);
1523
1524              /* Update vectorial force */
1525             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1526             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1527             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1528
1529             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1530             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1531             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1532
1533             /**************************
1534              * CALCULATE INTERACTIONS *
1535              **************************/
1536
1537             r01              = _mm_mul_ps(rsq01,rinv01);
1538
1539             /* Calculate table index by multiplying r with table scale and truncate to integer */
1540             rt               = _mm_mul_ps(r01,vftabscale);
1541             vfitab           = _mm_cvttps_epi32(rt);
1542 #ifdef __XOP__
1543             vfeps            = _mm_frcz_ps(rt);
1544 #else
1545             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1546 #endif
1547             twovfeps         = _mm_add_ps(vfeps,vfeps);
1548             vfitab           = _mm_slli_epi32(vfitab,2);
1549
1550             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1551             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1552             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1553             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1554             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1555             _MM_TRANSPOSE4_PS(Y,F,G,H);
1556             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1557             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1558             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
1559
1560             fscal            = felec;
1561
1562              /* Update vectorial force */
1563             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1564             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1565             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1566
1567             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1568             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1569             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1570
1571             /**************************
1572              * CALCULATE INTERACTIONS *
1573              **************************/
1574
1575             r02              = _mm_mul_ps(rsq02,rinv02);
1576
1577             /* Calculate table index by multiplying r with table scale and truncate to integer */
1578             rt               = _mm_mul_ps(r02,vftabscale);
1579             vfitab           = _mm_cvttps_epi32(rt);
1580 #ifdef __XOP__
1581             vfeps            = _mm_frcz_ps(rt);
1582 #else
1583             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1584 #endif
1585             twovfeps         = _mm_add_ps(vfeps,vfeps);
1586             vfitab           = _mm_slli_epi32(vfitab,2);
1587
1588             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1589             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1590             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1591             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1592             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1593             _MM_TRANSPOSE4_PS(Y,F,G,H);
1594             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1595             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1596             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
1597
1598             fscal            = felec;
1599
1600              /* Update vectorial force */
1601             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1602             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1603             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1604
1605             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1606             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1607             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1608
1609             /**************************
1610              * CALCULATE INTERACTIONS *
1611              **************************/
1612
1613             r10              = _mm_mul_ps(rsq10,rinv10);
1614
1615             /* Calculate table index by multiplying r with table scale and truncate to integer */
1616             rt               = _mm_mul_ps(r10,vftabscale);
1617             vfitab           = _mm_cvttps_epi32(rt);
1618 #ifdef __XOP__
1619             vfeps            = _mm_frcz_ps(rt);
1620 #else
1621             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1622 #endif
1623             twovfeps         = _mm_add_ps(vfeps,vfeps);
1624             vfitab           = _mm_slli_epi32(vfitab,2);
1625
1626             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1627             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1628             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1629             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1630             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1631             _MM_TRANSPOSE4_PS(Y,F,G,H);
1632             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1633             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1634             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
1635
1636             fscal            = felec;
1637
1638              /* Update vectorial force */
1639             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1640             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1641             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1642
1643             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1644             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1645             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1646
1647             /**************************
1648              * CALCULATE INTERACTIONS *
1649              **************************/
1650
1651             r11              = _mm_mul_ps(rsq11,rinv11);
1652
1653             /* Calculate table index by multiplying r with table scale and truncate to integer */
1654             rt               = _mm_mul_ps(r11,vftabscale);
1655             vfitab           = _mm_cvttps_epi32(rt);
1656 #ifdef __XOP__
1657             vfeps            = _mm_frcz_ps(rt);
1658 #else
1659             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1660 #endif
1661             twovfeps         = _mm_add_ps(vfeps,vfeps);
1662             vfitab           = _mm_slli_epi32(vfitab,2);
1663
1664             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1665             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1666             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1667             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1668             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1669             _MM_TRANSPOSE4_PS(Y,F,G,H);
1670             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1671             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1672             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
1673
1674             fscal            = felec;
1675
1676              /* Update vectorial force */
1677             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1678             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1679             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1680
1681             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1682             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1683             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1684
1685             /**************************
1686              * CALCULATE INTERACTIONS *
1687              **************************/
1688
1689             r12              = _mm_mul_ps(rsq12,rinv12);
1690
1691             /* Calculate table index by multiplying r with table scale and truncate to integer */
1692             rt               = _mm_mul_ps(r12,vftabscale);
1693             vfitab           = _mm_cvttps_epi32(rt);
1694 #ifdef __XOP__
1695             vfeps            = _mm_frcz_ps(rt);
1696 #else
1697             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1698 #endif
1699             twovfeps         = _mm_add_ps(vfeps,vfeps);
1700             vfitab           = _mm_slli_epi32(vfitab,2);
1701
1702             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1703             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1704             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1705             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1706             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1707             _MM_TRANSPOSE4_PS(Y,F,G,H);
1708             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1709             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1710             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
1711
1712             fscal            = felec;
1713
1714              /* Update vectorial force */
1715             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1716             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1717             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1718
1719             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1720             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1721             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1722
1723             /**************************
1724              * CALCULATE INTERACTIONS *
1725              **************************/
1726
1727             r20              = _mm_mul_ps(rsq20,rinv20);
1728
1729             /* Calculate table index by multiplying r with table scale and truncate to integer */
1730             rt               = _mm_mul_ps(r20,vftabscale);
1731             vfitab           = _mm_cvttps_epi32(rt);
1732 #ifdef __XOP__
1733             vfeps            = _mm_frcz_ps(rt);
1734 #else
1735             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1736 #endif
1737             twovfeps         = _mm_add_ps(vfeps,vfeps);
1738             vfitab           = _mm_slli_epi32(vfitab,2);
1739
1740             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1741             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1742             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1743             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1744             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1745             _MM_TRANSPOSE4_PS(Y,F,G,H);
1746             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1747             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1748             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1749
1750             fscal            = felec;
1751
1752              /* Update vectorial force */
1753             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1754             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1755             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1756
1757             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1758             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1759             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1760
1761             /**************************
1762              * CALCULATE INTERACTIONS *
1763              **************************/
1764
1765             r21              = _mm_mul_ps(rsq21,rinv21);
1766
1767             /* Calculate table index by multiplying r with table scale and truncate to integer */
1768             rt               = _mm_mul_ps(r21,vftabscale);
1769             vfitab           = _mm_cvttps_epi32(rt);
1770 #ifdef __XOP__
1771             vfeps            = _mm_frcz_ps(rt);
1772 #else
1773             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1774 #endif
1775             twovfeps         = _mm_add_ps(vfeps,vfeps);
1776             vfitab           = _mm_slli_epi32(vfitab,2);
1777
1778             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1779             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1780             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1781             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1782             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1783             _MM_TRANSPOSE4_PS(Y,F,G,H);
1784             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1785             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1786             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
1787
1788             fscal            = felec;
1789
1790              /* Update vectorial force */
1791             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1792             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1793             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1794
1795             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1796             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1797             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1798
1799             /**************************
1800              * CALCULATE INTERACTIONS *
1801              **************************/
1802
1803             r22              = _mm_mul_ps(rsq22,rinv22);
1804
1805             /* Calculate table index by multiplying r with table scale and truncate to integer */
1806             rt               = _mm_mul_ps(r22,vftabscale);
1807             vfitab           = _mm_cvttps_epi32(rt);
1808 #ifdef __XOP__
1809             vfeps            = _mm_frcz_ps(rt);
1810 #else
1811             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1812 #endif
1813             twovfeps         = _mm_add_ps(vfeps,vfeps);
1814             vfitab           = _mm_slli_epi32(vfitab,2);
1815
1816             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1817             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1818             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1819             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1820             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1821             _MM_TRANSPOSE4_PS(Y,F,G,H);
1822             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1823             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1824             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
1825
1826             fscal            = felec;
1827
1828              /* Update vectorial force */
1829             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1830             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1831             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1832
1833             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1834             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1835             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1836
1837             fjptrA             = f+j_coord_offsetA;
1838             fjptrB             = f+j_coord_offsetB;
1839             fjptrC             = f+j_coord_offsetC;
1840             fjptrD             = f+j_coord_offsetD;
1841
1842             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1843                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1844
1845             /* Inner loop uses 386 flops */
1846         }
1847
1848         if(jidx<j_index_end)
1849         {
1850
1851             /* Get j neighbor index, and coordinate index */
1852             jnrlistA         = jjnr[jidx];
1853             jnrlistB         = jjnr[jidx+1];
1854             jnrlistC         = jjnr[jidx+2];
1855             jnrlistD         = jjnr[jidx+3];
1856             /* Sign of each element will be negative for non-real atoms.
1857              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1858              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1859              */
1860             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1861             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
1862             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
1863             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1864             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1865             j_coord_offsetA  = DIM*jnrA;
1866             j_coord_offsetB  = DIM*jnrB;
1867             j_coord_offsetC  = DIM*jnrC;
1868             j_coord_offsetD  = DIM*jnrD;
1869
1870             /* load j atom coordinates */
1871             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1872                                               x+j_coord_offsetC,x+j_coord_offsetD,
1873                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1874
1875             /* Calculate displacement vector */
1876             dx00             = _mm_sub_ps(ix0,jx0);
1877             dy00             = _mm_sub_ps(iy0,jy0);
1878             dz00             = _mm_sub_ps(iz0,jz0);
1879             dx01             = _mm_sub_ps(ix0,jx1);
1880             dy01             = _mm_sub_ps(iy0,jy1);
1881             dz01             = _mm_sub_ps(iz0,jz1);
1882             dx02             = _mm_sub_ps(ix0,jx2);
1883             dy02             = _mm_sub_ps(iy0,jy2);
1884             dz02             = _mm_sub_ps(iz0,jz2);
1885             dx10             = _mm_sub_ps(ix1,jx0);
1886             dy10             = _mm_sub_ps(iy1,jy0);
1887             dz10             = _mm_sub_ps(iz1,jz0);
1888             dx11             = _mm_sub_ps(ix1,jx1);
1889             dy11             = _mm_sub_ps(iy1,jy1);
1890             dz11             = _mm_sub_ps(iz1,jz1);
1891             dx12             = _mm_sub_ps(ix1,jx2);
1892             dy12             = _mm_sub_ps(iy1,jy2);
1893             dz12             = _mm_sub_ps(iz1,jz2);
1894             dx20             = _mm_sub_ps(ix2,jx0);
1895             dy20             = _mm_sub_ps(iy2,jy0);
1896             dz20             = _mm_sub_ps(iz2,jz0);
1897             dx21             = _mm_sub_ps(ix2,jx1);
1898             dy21             = _mm_sub_ps(iy2,jy1);
1899             dz21             = _mm_sub_ps(iz2,jz1);
1900             dx22             = _mm_sub_ps(ix2,jx2);
1901             dy22             = _mm_sub_ps(iy2,jy2);
1902             dz22             = _mm_sub_ps(iz2,jz2);
1903
1904             /* Calculate squared distance and things based on it */
1905             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1906             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1907             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1908             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1909             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1910             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1911             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1912             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1913             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1914
1915             rinv00           = avx128fma_invsqrt_f(rsq00);
1916             rinv01           = avx128fma_invsqrt_f(rsq01);
1917             rinv02           = avx128fma_invsqrt_f(rsq02);
1918             rinv10           = avx128fma_invsqrt_f(rsq10);
1919             rinv11           = avx128fma_invsqrt_f(rsq11);
1920             rinv12           = avx128fma_invsqrt_f(rsq12);
1921             rinv20           = avx128fma_invsqrt_f(rsq20);
1922             rinv21           = avx128fma_invsqrt_f(rsq21);
1923             rinv22           = avx128fma_invsqrt_f(rsq22);
1924
1925             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1926
1927             fjx0             = _mm_setzero_ps();
1928             fjy0             = _mm_setzero_ps();
1929             fjz0             = _mm_setzero_ps();
1930             fjx1             = _mm_setzero_ps();
1931             fjy1             = _mm_setzero_ps();
1932             fjz1             = _mm_setzero_ps();
1933             fjx2             = _mm_setzero_ps();
1934             fjy2             = _mm_setzero_ps();
1935             fjz2             = _mm_setzero_ps();
1936
1937             /**************************
1938              * CALCULATE INTERACTIONS *
1939              **************************/
1940
1941             r00              = _mm_mul_ps(rsq00,rinv00);
1942             r00              = _mm_andnot_ps(dummy_mask,r00);
1943
1944             /* Calculate table index by multiplying r with table scale and truncate to integer */
1945             rt               = _mm_mul_ps(r00,vftabscale);
1946             vfitab           = _mm_cvttps_epi32(rt);
1947 #ifdef __XOP__
1948             vfeps            = _mm_frcz_ps(rt);
1949 #else
1950             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1951 #endif
1952             twovfeps         = _mm_add_ps(vfeps,vfeps);
1953             vfitab           = _mm_slli_epi32(vfitab,2);
1954
1955             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1956             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1957             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1958             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1959             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1960             _MM_TRANSPOSE4_PS(Y,F,G,H);
1961             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1962             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1963             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq00,FF),_mm_mul_ps(vftabscale,rinv00)));
1964
1965             /* LENNARD-JONES DISPERSION/REPULSION */
1966
1967             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1968             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1969
1970             fscal            = _mm_add_ps(felec,fvdw);
1971
1972             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1973
1974              /* Update vectorial force */
1975             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1976             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1977             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1978
1979             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1980             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1981             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1982
1983             /**************************
1984              * CALCULATE INTERACTIONS *
1985              **************************/
1986
1987             r01              = _mm_mul_ps(rsq01,rinv01);
1988             r01              = _mm_andnot_ps(dummy_mask,r01);
1989
1990             /* Calculate table index by multiplying r with table scale and truncate to integer */
1991             rt               = _mm_mul_ps(r01,vftabscale);
1992             vfitab           = _mm_cvttps_epi32(rt);
1993 #ifdef __XOP__
1994             vfeps            = _mm_frcz_ps(rt);
1995 #else
1996             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1997 #endif
1998             twovfeps         = _mm_add_ps(vfeps,vfeps);
1999             vfitab           = _mm_slli_epi32(vfitab,2);
2000
2001             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2002             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2003             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2004             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2005             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2006             _MM_TRANSPOSE4_PS(Y,F,G,H);
2007             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2008             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2009             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq01,FF),_mm_mul_ps(vftabscale,rinv01)));
2010
2011             fscal            = felec;
2012
2013             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2014
2015              /* Update vectorial force */
2016             fix0             = _mm_macc_ps(dx01,fscal,fix0);
2017             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
2018             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
2019
2020             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
2021             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
2022             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
2023
2024             /**************************
2025              * CALCULATE INTERACTIONS *
2026              **************************/
2027
2028             r02              = _mm_mul_ps(rsq02,rinv02);
2029             r02              = _mm_andnot_ps(dummy_mask,r02);
2030
2031             /* Calculate table index by multiplying r with table scale and truncate to integer */
2032             rt               = _mm_mul_ps(r02,vftabscale);
2033             vfitab           = _mm_cvttps_epi32(rt);
2034 #ifdef __XOP__
2035             vfeps            = _mm_frcz_ps(rt);
2036 #else
2037             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2038 #endif
2039             twovfeps         = _mm_add_ps(vfeps,vfeps);
2040             vfitab           = _mm_slli_epi32(vfitab,2);
2041
2042             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2043             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2044             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2045             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2046             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2047             _MM_TRANSPOSE4_PS(Y,F,G,H);
2048             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2049             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2050             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq02,FF),_mm_mul_ps(vftabscale,rinv02)));
2051
2052             fscal            = felec;
2053
2054             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2055
2056              /* Update vectorial force */
2057             fix0             = _mm_macc_ps(dx02,fscal,fix0);
2058             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
2059             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
2060
2061             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
2062             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
2063             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
2064
2065             /**************************
2066              * CALCULATE INTERACTIONS *
2067              **************************/
2068
2069             r10              = _mm_mul_ps(rsq10,rinv10);
2070             r10              = _mm_andnot_ps(dummy_mask,r10);
2071
2072             /* Calculate table index by multiplying r with table scale and truncate to integer */
2073             rt               = _mm_mul_ps(r10,vftabscale);
2074             vfitab           = _mm_cvttps_epi32(rt);
2075 #ifdef __XOP__
2076             vfeps            = _mm_frcz_ps(rt);
2077 #else
2078             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2079 #endif
2080             twovfeps         = _mm_add_ps(vfeps,vfeps);
2081             vfitab           = _mm_slli_epi32(vfitab,2);
2082
2083             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2084             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2085             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2086             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2087             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2088             _MM_TRANSPOSE4_PS(Y,F,G,H);
2089             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2090             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2091             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
2092
2093             fscal            = felec;
2094
2095             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2096
2097              /* Update vectorial force */
2098             fix1             = _mm_macc_ps(dx10,fscal,fix1);
2099             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
2100             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
2101
2102             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
2103             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
2104             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
2105
2106             /**************************
2107              * CALCULATE INTERACTIONS *
2108              **************************/
2109
2110             r11              = _mm_mul_ps(rsq11,rinv11);
2111             r11              = _mm_andnot_ps(dummy_mask,r11);
2112
2113             /* Calculate table index by multiplying r with table scale and truncate to integer */
2114             rt               = _mm_mul_ps(r11,vftabscale);
2115             vfitab           = _mm_cvttps_epi32(rt);
2116 #ifdef __XOP__
2117             vfeps            = _mm_frcz_ps(rt);
2118 #else
2119             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2120 #endif
2121             twovfeps         = _mm_add_ps(vfeps,vfeps);
2122             vfitab           = _mm_slli_epi32(vfitab,2);
2123
2124             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2125             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2126             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2127             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2128             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2129             _MM_TRANSPOSE4_PS(Y,F,G,H);
2130             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2131             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2132             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq11,FF),_mm_mul_ps(vftabscale,rinv11)));
2133
2134             fscal            = felec;
2135
2136             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2137
2138              /* Update vectorial force */
2139             fix1             = _mm_macc_ps(dx11,fscal,fix1);
2140             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
2141             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
2142
2143             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
2144             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
2145             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
2146
2147             /**************************
2148              * CALCULATE INTERACTIONS *
2149              **************************/
2150
2151             r12              = _mm_mul_ps(rsq12,rinv12);
2152             r12              = _mm_andnot_ps(dummy_mask,r12);
2153
2154             /* Calculate table index by multiplying r with table scale and truncate to integer */
2155             rt               = _mm_mul_ps(r12,vftabscale);
2156             vfitab           = _mm_cvttps_epi32(rt);
2157 #ifdef __XOP__
2158             vfeps            = _mm_frcz_ps(rt);
2159 #else
2160             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2161 #endif
2162             twovfeps         = _mm_add_ps(vfeps,vfeps);
2163             vfitab           = _mm_slli_epi32(vfitab,2);
2164
2165             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2166             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2167             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2168             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2169             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2170             _MM_TRANSPOSE4_PS(Y,F,G,H);
2171             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2172             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2173             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq12,FF),_mm_mul_ps(vftabscale,rinv12)));
2174
2175             fscal            = felec;
2176
2177             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2178
2179              /* Update vectorial force */
2180             fix1             = _mm_macc_ps(dx12,fscal,fix1);
2181             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
2182             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
2183
2184             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
2185             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
2186             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
2187
2188             /**************************
2189              * CALCULATE INTERACTIONS *
2190              **************************/
2191
2192             r20              = _mm_mul_ps(rsq20,rinv20);
2193             r20              = _mm_andnot_ps(dummy_mask,r20);
2194
2195             /* Calculate table index by multiplying r with table scale and truncate to integer */
2196             rt               = _mm_mul_ps(r20,vftabscale);
2197             vfitab           = _mm_cvttps_epi32(rt);
2198 #ifdef __XOP__
2199             vfeps            = _mm_frcz_ps(rt);
2200 #else
2201             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2202 #endif
2203             twovfeps         = _mm_add_ps(vfeps,vfeps);
2204             vfitab           = _mm_slli_epi32(vfitab,2);
2205
2206             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2207             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2208             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2209             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2210             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2211             _MM_TRANSPOSE4_PS(Y,F,G,H);
2212             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2213             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2214             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
2215
2216             fscal            = felec;
2217
2218             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2219
2220              /* Update vectorial force */
2221             fix2             = _mm_macc_ps(dx20,fscal,fix2);
2222             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
2223             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
2224
2225             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
2226             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
2227             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
2228
2229             /**************************
2230              * CALCULATE INTERACTIONS *
2231              **************************/
2232
2233             r21              = _mm_mul_ps(rsq21,rinv21);
2234             r21              = _mm_andnot_ps(dummy_mask,r21);
2235
2236             /* Calculate table index by multiplying r with table scale and truncate to integer */
2237             rt               = _mm_mul_ps(r21,vftabscale);
2238             vfitab           = _mm_cvttps_epi32(rt);
2239 #ifdef __XOP__
2240             vfeps            = _mm_frcz_ps(rt);
2241 #else
2242             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2243 #endif
2244             twovfeps         = _mm_add_ps(vfeps,vfeps);
2245             vfitab           = _mm_slli_epi32(vfitab,2);
2246
2247             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2248             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2249             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2250             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2251             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2252             _MM_TRANSPOSE4_PS(Y,F,G,H);
2253             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2254             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2255             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq21,FF),_mm_mul_ps(vftabscale,rinv21)));
2256
2257             fscal            = felec;
2258
2259             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2260
2261              /* Update vectorial force */
2262             fix2             = _mm_macc_ps(dx21,fscal,fix2);
2263             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
2264             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
2265
2266             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
2267             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
2268             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
2269
2270             /**************************
2271              * CALCULATE INTERACTIONS *
2272              **************************/
2273
2274             r22              = _mm_mul_ps(rsq22,rinv22);
2275             r22              = _mm_andnot_ps(dummy_mask,r22);
2276
2277             /* Calculate table index by multiplying r with table scale and truncate to integer */
2278             rt               = _mm_mul_ps(r22,vftabscale);
2279             vfitab           = _mm_cvttps_epi32(rt);
2280 #ifdef __XOP__
2281             vfeps            = _mm_frcz_ps(rt);
2282 #else
2283             vfeps            = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
2284 #endif
2285             twovfeps         = _mm_add_ps(vfeps,vfeps);
2286             vfitab           = _mm_slli_epi32(vfitab,2);
2287
2288             /* CUBIC SPLINE TABLE ELECTROSTATICS */
2289             Y                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
2290             F                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
2291             G                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
2292             H                = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
2293             _MM_TRANSPOSE4_PS(Y,F,G,H);
2294             Fp               = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
2295             FF               = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
2296             felec            = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq22,FF),_mm_mul_ps(vftabscale,rinv22)));
2297
2298             fscal            = felec;
2299
2300             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2301
2302              /* Update vectorial force */
2303             fix2             = _mm_macc_ps(dx22,fscal,fix2);
2304             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
2305             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
2306
2307             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
2308             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
2309             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
2310
2311             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
2312             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
2313             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
2314             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
2315
2316             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
2317                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
2318
2319             /* Inner loop uses 395 flops */
2320         }
2321
2322         /* End of innermost loop */
2323
2324         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
2325                                               f+i_coord_offset,fshift+i_shift_offset);
2326
2327         /* Increment number of inner iterations */
2328         inneriter                  += j_index_end - j_index_start;
2329
2330         /* Outer loop uses 18 flops */
2331     }
2332
2333     /* Increment number of outer iterations */
2334     outeriter        += nri;
2335
2336     /* Update outer/inner flops */
2337
2338     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*395);
2339 }