src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecEw_VdwLJ_GeomW4W4_avx_128_fma_single.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014,2015,2017, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
  37  */
  38 #include "gmxpre.h"
  39
  40 #include "config.h"
  41
  42 #include <math.h>
  43
  44 #include "../nb_kernel.h"
  45 #include "gromacs/gmxlib/nrnb.h"
  46
  47 #include "kernelutil_x86_avx_128_fma_single.h"
  48
  49 /*
  50  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4W4_VF_avx_128_fma_single
  51  * Electrostatics interaction: Ewald
  52  * VdW interaction:            LennardJones
  53  * Geometry:                   Water4-Water4
  54  * Calculate force/pot:        PotentialAndForce
  55  */
  56 void
  57 nb_kernel_ElecEw_VdwLJ_GeomW4W4_VF_avx_128_fma_single
  58                     (t_nblist                    * gmx_restrict       nlist,
  59                      rvec                        * gmx_restrict          xx,
  60                      rvec                        * gmx_restrict          ff,
  61                      struct t_forcerec           * gmx_restrict          fr,
  62                      t_mdatoms                   * gmx_restrict     mdatoms,
  63                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  64                      t_nrnb                      * gmx_restrict        nrnb)
  65 {
  66     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  67      * just 0 for non-waters.
  68      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  69      * jnr indices corresponding to data put in the four positions in the SIMD register.
  70      */
  71     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  72     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  73     int              jnrA,jnrB,jnrC,jnrD;
  74     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  75     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  76     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  77     real             rcutoff_scalar;
  78     real             *shiftvec,*fshift,*x,*f;
  79     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  80     real             scratch[4*DIM];
  81     __m128           fscal,rcutoff,rcutoff2,jidxall;
  82     int              vdwioffset0;
  83     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  84     int              vdwioffset1;
  85     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  86     int              vdwioffset2;
  87     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  88     int              vdwioffset3;
  89     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
  90     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  91     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  92     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
  93     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  94     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
  95     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  96     int              vdwjidx3A,vdwjidx3B,vdwjidx3C,vdwjidx3D;
  97     __m128           jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
  98     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  99     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 100     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 101     __m128           dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
 102     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 103     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 104     __m128           dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
 105     __m128           dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
 106     __m128           dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
 107     __m128           dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
 108     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 109     real             *charge;
 110     int              nvdwtype;
 111     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 112     int              *vdwtype;
 113     real             *vdwparam;
 114     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 115     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 116     __m128i          ewitab;
 117     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
 118     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
 119     real             *ewtab;
 120     __m128           dummy_mask,cutoff_mask;
 121     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 122     __m128           one     = _mm_set1_ps(1.0);
 123     __m128           two     = _mm_set1_ps(2.0);
 124     x                = xx[0];
 125     f                = ff[0];
 126
 127     nri              = nlist->nri;
 128     iinr             = nlist->iinr;
 129     jindex           = nlist->jindex;
 130     jjnr             = nlist->jjnr;
 131     shiftidx         = nlist->shift;
 132     gid              = nlist->gid;
 133     shiftvec         = fr->shift_vec[0];
 134     fshift           = fr->fshift[0];
 135     facel            = _mm_set1_ps(fr->ic->epsfac);
 136     charge           = mdatoms->chargeA;
 137     nvdwtype         = fr->ntype;
 138     vdwparam         = fr->nbfp;
 139     vdwtype          = mdatoms->typeA;
 140
 141     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
 142     beta             = _mm_set1_ps(fr->ic->ewaldcoeff_q);
 143     beta2            = _mm_mul_ps(beta,beta);
 144     beta3            = _mm_mul_ps(beta,beta2);
 145     ewtab            = fr->ic->tabq_coul_FDV0;
 146     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
 147     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
 148
 149     /* Setup water-specific parameters */
 150     inr              = nlist->iinr[0];
 151     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 152     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 153     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
 154     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 155
 156     jq1              = _mm_set1_ps(charge[inr+1]);
 157     jq2              = _mm_set1_ps(charge[inr+2]);
 158     jq3              = _mm_set1_ps(charge[inr+3]);
 159     vdwjidx0A        = 2*vdwtype[inr+0];
 160     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 161     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 162     qq11             = _mm_mul_ps(iq1,jq1);
 163     qq12             = _mm_mul_ps(iq1,jq2);
 164     qq13             = _mm_mul_ps(iq1,jq3);
 165     qq21             = _mm_mul_ps(iq2,jq1);
 166     qq22             = _mm_mul_ps(iq2,jq2);
 167     qq23             = _mm_mul_ps(iq2,jq3);
 168     qq31             = _mm_mul_ps(iq3,jq1);
 169     qq32             = _mm_mul_ps(iq3,jq2);
 170     qq33             = _mm_mul_ps(iq3,jq3);
 171
 172     /* Avoid stupid compiler warnings */
 173     jnrA = jnrB = jnrC = jnrD = 0;
 174     j_coord_offsetA = 0;
 175     j_coord_offsetB = 0;
 176     j_coord_offsetC = 0;
 177     j_coord_offsetD = 0;
 178
 179     outeriter        = 0;
 180     inneriter        = 0;
 181
 182     for(iidx=0;iidx<4*DIM;iidx++)
 183     {
 184         scratch[iidx] = 0.0;
 185     }
 186
 187     /* Start outer loop over neighborlists */
 188     for(iidx=0; iidx<nri; iidx++)
 189     {
 190         /* Load shift vector for this list */
 191         i_shift_offset   = DIM*shiftidx[iidx];
 192
 193         /* Load limits for loop over neighbors */
 194         j_index_start    = jindex[iidx];
 195         j_index_end      = jindex[iidx+1];
 196
 197         /* Get outer coordinate index */
 198         inr              = iinr[iidx];
 199         i_coord_offset   = DIM*inr;
 200
 201         /* Load i particle coords and add shift vector */
 202         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 203                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 204
 205         fix0             = _mm_setzero_ps();
 206         fiy0             = _mm_setzero_ps();
 207         fiz0             = _mm_setzero_ps();
 208         fix1             = _mm_setzero_ps();
 209         fiy1             = _mm_setzero_ps();
 210         fiz1             = _mm_setzero_ps();
 211         fix2             = _mm_setzero_ps();
 212         fiy2             = _mm_setzero_ps();
 213         fiz2             = _mm_setzero_ps();
 214         fix3             = _mm_setzero_ps();
 215         fiy3             = _mm_setzero_ps();
 216         fiz3             = _mm_setzero_ps();
 217
 218         /* Reset potential sums */
 219         velecsum         = _mm_setzero_ps();
 220         vvdwsum          = _mm_setzero_ps();
 221
 222         /* Start inner kernel loop */
 223         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 224         {
 225
 226             /* Get j neighbor index, and coordinate index */
 227             jnrA             = jjnr[jidx];
 228             jnrB             = jjnr[jidx+1];
 229             jnrC             = jjnr[jidx+2];
 230             jnrD             = jjnr[jidx+3];
 231             j_coord_offsetA  = DIM*jnrA;
 232             j_coord_offsetB  = DIM*jnrB;
 233             j_coord_offsetC  = DIM*jnrC;
 234             j_coord_offsetD  = DIM*jnrD;
 235
 236             /* load j atom coordinates */
 237             gmx_mm_load_4rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 238                                               x+j_coord_offsetC,x+j_coord_offsetD,
 239                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
 240                                               &jy2,&jz2,&jx3,&jy3,&jz3);
 241
 242             /* Calculate displacement vector */
 243             dx00             = _mm_sub_ps(ix0,jx0);
 244             dy00             = _mm_sub_ps(iy0,jy0);
 245             dz00             = _mm_sub_ps(iz0,jz0);
 246             dx11             = _mm_sub_ps(ix1,jx1);
 247             dy11             = _mm_sub_ps(iy1,jy1);
 248             dz11             = _mm_sub_ps(iz1,jz1);
 249             dx12             = _mm_sub_ps(ix1,jx2);
 250             dy12             = _mm_sub_ps(iy1,jy2);
 251             dz12             = _mm_sub_ps(iz1,jz2);
 252             dx13             = _mm_sub_ps(ix1,jx3);
 253             dy13             = _mm_sub_ps(iy1,jy3);
 254             dz13             = _mm_sub_ps(iz1,jz3);
 255             dx21             = _mm_sub_ps(ix2,jx1);
 256             dy21             = _mm_sub_ps(iy2,jy1);
 257             dz21             = _mm_sub_ps(iz2,jz1);
 258             dx22             = _mm_sub_ps(ix2,jx2);
 259             dy22             = _mm_sub_ps(iy2,jy2);
 260             dz22             = _mm_sub_ps(iz2,jz2);
 261             dx23             = _mm_sub_ps(ix2,jx3);
 262             dy23             = _mm_sub_ps(iy2,jy3);
 263             dz23             = _mm_sub_ps(iz2,jz3);
 264             dx31             = _mm_sub_ps(ix3,jx1);
 265             dy31             = _mm_sub_ps(iy3,jy1);
 266             dz31             = _mm_sub_ps(iz3,jz1);
 267             dx32             = _mm_sub_ps(ix3,jx2);
 268             dy32             = _mm_sub_ps(iy3,jy2);
 269             dz32             = _mm_sub_ps(iz3,jz2);
 270             dx33             = _mm_sub_ps(ix3,jx3);
 271             dy33             = _mm_sub_ps(iy3,jy3);
 272             dz33             = _mm_sub_ps(iz3,jz3);
 273
 274             /* Calculate squared distance and things based on it */
 275             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 276             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 277             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 278             rsq13            = gmx_mm_calc_rsq_ps(dx13,dy13,dz13);
 279             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 280             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 281             rsq23            = gmx_mm_calc_rsq_ps(dx23,dy23,dz23);
 282             rsq31            = gmx_mm_calc_rsq_ps(dx31,dy31,dz31);
 283             rsq32            = gmx_mm_calc_rsq_ps(dx32,dy32,dz32);
 284             rsq33            = gmx_mm_calc_rsq_ps(dx33,dy33,dz33);
 285
 286             rinv11           = avx128fma_invsqrt_f(rsq11);
 287             rinv12           = avx128fma_invsqrt_f(rsq12);
 288             rinv13           = avx128fma_invsqrt_f(rsq13);
 289             rinv21           = avx128fma_invsqrt_f(rsq21);
 290             rinv22           = avx128fma_invsqrt_f(rsq22);
 291             rinv23           = avx128fma_invsqrt_f(rsq23);
 292             rinv31           = avx128fma_invsqrt_f(rsq31);
 293             rinv32           = avx128fma_invsqrt_f(rsq32);
 294             rinv33           = avx128fma_invsqrt_f(rsq33);
 295
 296             rinvsq00         = avx128fma_inv_f(rsq00);
 297             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 298             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 299             rinvsq13         = _mm_mul_ps(rinv13,rinv13);
 300             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 301             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 302             rinvsq23         = _mm_mul_ps(rinv23,rinv23);
 303             rinvsq31         = _mm_mul_ps(rinv31,rinv31);
 304             rinvsq32         = _mm_mul_ps(rinv32,rinv32);
 305             rinvsq33         = _mm_mul_ps(rinv33,rinv33);
 306
 307             fjx0             = _mm_setzero_ps();
 308             fjy0             = _mm_setzero_ps();
 309             fjz0             = _mm_setzero_ps();
 310             fjx1             = _mm_setzero_ps();
 311             fjy1             = _mm_setzero_ps();
 312             fjz1             = _mm_setzero_ps();
 313             fjx2             = _mm_setzero_ps();
 314             fjy2             = _mm_setzero_ps();
 315             fjz2             = _mm_setzero_ps();
 316             fjx3             = _mm_setzero_ps();
 317             fjy3             = _mm_setzero_ps();
 318             fjz3             = _mm_setzero_ps();
 319
 320             /**************************
 321              * CALCULATE INTERACTIONS *
 322              **************************/
 323
 324             /* LENNARD-JONES DISPERSION/REPULSION */
 325
 326             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 327             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 328             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 329             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 330             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 331
 332             /* Update potential sum for this i atom from the interaction with this j atom. */
 333             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 334
 335             fscal            = fvdw;
 336
 337              /* Update vectorial force */
 338             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 339             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 340             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 341
 342             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 343             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 344             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 345
 346             /**************************
 347              * CALCULATE INTERACTIONS *
 348              **************************/
 349
 350             r11              = _mm_mul_ps(rsq11,rinv11);
 351
 352             /* EWALD ELECTROSTATICS */
 353
 354             /* Analytical PME correction */
 355             zeta2            = _mm_mul_ps(beta2,rsq11);
 356             rinv3            = _mm_mul_ps(rinvsq11,rinv11);
 357             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 358             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 359             felec            = _mm_mul_ps(qq11,felec);
 360             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 361             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv11);
 362             velec            = _mm_mul_ps(qq11,velec);
 363
 364             /* Update potential sum for this i atom from the interaction with this j atom. */
 365             velecsum         = _mm_add_ps(velecsum,velec);
 366
 367             fscal            = felec;
 368
 369              /* Update vectorial force */
 370             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 371             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 372             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 373
 374             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 375             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 376             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 377
 378             /**************************
 379              * CALCULATE INTERACTIONS *
 380              **************************/
 381
 382             r12              = _mm_mul_ps(rsq12,rinv12);
 383
 384             /* EWALD ELECTROSTATICS */
 385
 386             /* Analytical PME correction */
 387             zeta2            = _mm_mul_ps(beta2,rsq12);
 388             rinv3            = _mm_mul_ps(rinvsq12,rinv12);
 389             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 390             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 391             felec            = _mm_mul_ps(qq12,felec);
 392             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 393             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv12);
 394             velec            = _mm_mul_ps(qq12,velec);
 395
 396             /* Update potential sum for this i atom from the interaction with this j atom. */
 397             velecsum         = _mm_add_ps(velecsum,velec);
 398
 399             fscal            = felec;
 400
 401              /* Update vectorial force */
 402             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 403             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 404             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 405
 406             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 407             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 408             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 409
 410             /**************************
 411              * CALCULATE INTERACTIONS *
 412              **************************/
 413
 414             r13              = _mm_mul_ps(rsq13,rinv13);
 415
 416             /* EWALD ELECTROSTATICS */
 417
 418             /* Analytical PME correction */
 419             zeta2            = _mm_mul_ps(beta2,rsq13);
 420             rinv3            = _mm_mul_ps(rinvsq13,rinv13);
 421             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 422             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 423             felec            = _mm_mul_ps(qq13,felec);
 424             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 425             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv13);
 426             velec            = _mm_mul_ps(qq13,velec);
 427
 428             /* Update potential sum for this i atom from the interaction with this j atom. */
 429             velecsum         = _mm_add_ps(velecsum,velec);
 430
 431             fscal            = felec;
 432
 433              /* Update vectorial force */
 434             fix1             = _mm_macc_ps(dx13,fscal,fix1);
 435             fiy1             = _mm_macc_ps(dy13,fscal,fiy1);
 436             fiz1             = _mm_macc_ps(dz13,fscal,fiz1);
 437
 438             fjx3             = _mm_macc_ps(dx13,fscal,fjx3);
 439             fjy3             = _mm_macc_ps(dy13,fscal,fjy3);
 440             fjz3             = _mm_macc_ps(dz13,fscal,fjz3);
 441
 442             /**************************
 443              * CALCULATE INTERACTIONS *
 444              **************************/
 445
 446             r21              = _mm_mul_ps(rsq21,rinv21);
 447
 448             /* EWALD ELECTROSTATICS */
 449
 450             /* Analytical PME correction */
 451             zeta2            = _mm_mul_ps(beta2,rsq21);
 452             rinv3            = _mm_mul_ps(rinvsq21,rinv21);
 453             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 454             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 455             felec            = _mm_mul_ps(qq21,felec);
 456             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 457             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv21);
 458             velec            = _mm_mul_ps(qq21,velec);
 459
 460             /* Update potential sum for this i atom from the interaction with this j atom. */
 461             velecsum         = _mm_add_ps(velecsum,velec);
 462
 463             fscal            = felec;
 464
 465              /* Update vectorial force */
 466             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 467             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 468             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 469
 470             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 471             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 472             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 473
 474             /**************************
 475              * CALCULATE INTERACTIONS *
 476              **************************/
 477
 478             r22              = _mm_mul_ps(rsq22,rinv22);
 479
 480             /* EWALD ELECTROSTATICS */
 481
 482             /* Analytical PME correction */
 483             zeta2            = _mm_mul_ps(beta2,rsq22);
 484             rinv3            = _mm_mul_ps(rinvsq22,rinv22);
 485             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 486             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 487             felec            = _mm_mul_ps(qq22,felec);
 488             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 489             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv22);
 490             velec            = _mm_mul_ps(qq22,velec);
 491
 492             /* Update potential sum for this i atom from the interaction with this j atom. */
 493             velecsum         = _mm_add_ps(velecsum,velec);
 494
 495             fscal            = felec;
 496
 497              /* Update vectorial force */
 498             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 499             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 500             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 501
 502             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 503             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 504             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 505
 506             /**************************
 507              * CALCULATE INTERACTIONS *
 508              **************************/
 509
 510             r23              = _mm_mul_ps(rsq23,rinv23);
 511
 512             /* EWALD ELECTROSTATICS */
 513
 514             /* Analytical PME correction */
 515             zeta2            = _mm_mul_ps(beta2,rsq23);
 516             rinv3            = _mm_mul_ps(rinvsq23,rinv23);
 517             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 518             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 519             felec            = _mm_mul_ps(qq23,felec);
 520             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 521             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv23);
 522             velec            = _mm_mul_ps(qq23,velec);
 523
 524             /* Update potential sum for this i atom from the interaction with this j atom. */
 525             velecsum         = _mm_add_ps(velecsum,velec);
 526
 527             fscal            = felec;
 528
 529              /* Update vectorial force */
 530             fix2             = _mm_macc_ps(dx23,fscal,fix2);
 531             fiy2             = _mm_macc_ps(dy23,fscal,fiy2);
 532             fiz2             = _mm_macc_ps(dz23,fscal,fiz2);
 533
 534             fjx3             = _mm_macc_ps(dx23,fscal,fjx3);
 535             fjy3             = _mm_macc_ps(dy23,fscal,fjy3);
 536             fjz3             = _mm_macc_ps(dz23,fscal,fjz3);
 537
 538             /**************************
 539              * CALCULATE INTERACTIONS *
 540              **************************/
 541
 542             r31              = _mm_mul_ps(rsq31,rinv31);
 543
 544             /* EWALD ELECTROSTATICS */
 545
 546             /* Analytical PME correction */
 547             zeta2            = _mm_mul_ps(beta2,rsq31);
 548             rinv3            = _mm_mul_ps(rinvsq31,rinv31);
 549             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 550             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 551             felec            = _mm_mul_ps(qq31,felec);
 552             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 553             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv31);
 554             velec            = _mm_mul_ps(qq31,velec);
 555
 556             /* Update potential sum for this i atom from the interaction with this j atom. */
 557             velecsum         = _mm_add_ps(velecsum,velec);
 558
 559             fscal            = felec;
 560
 561              /* Update vectorial force */
 562             fix3             = _mm_macc_ps(dx31,fscal,fix3);
 563             fiy3             = _mm_macc_ps(dy31,fscal,fiy3);
 564             fiz3             = _mm_macc_ps(dz31,fscal,fiz3);
 565
 566             fjx1             = _mm_macc_ps(dx31,fscal,fjx1);
 567             fjy1             = _mm_macc_ps(dy31,fscal,fjy1);
 568             fjz1             = _mm_macc_ps(dz31,fscal,fjz1);
 569
 570             /**************************
 571              * CALCULATE INTERACTIONS *
 572              **************************/
 573
 574             r32              = _mm_mul_ps(rsq32,rinv32);
 575
 576             /* EWALD ELECTROSTATICS */
 577
 578             /* Analytical PME correction */
 579             zeta2            = _mm_mul_ps(beta2,rsq32);
 580             rinv3            = _mm_mul_ps(rinvsq32,rinv32);
 581             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 582             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 583             felec            = _mm_mul_ps(qq32,felec);
 584             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 585             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv32);
 586             velec            = _mm_mul_ps(qq32,velec);
 587
 588             /* Update potential sum for this i atom from the interaction with this j atom. */
 589             velecsum         = _mm_add_ps(velecsum,velec);
 590
 591             fscal            = felec;
 592
 593              /* Update vectorial force */
 594             fix3             = _mm_macc_ps(dx32,fscal,fix3);
 595             fiy3             = _mm_macc_ps(dy32,fscal,fiy3);
 596             fiz3             = _mm_macc_ps(dz32,fscal,fiz3);
 597
 598             fjx2             = _mm_macc_ps(dx32,fscal,fjx2);
 599             fjy2             = _mm_macc_ps(dy32,fscal,fjy2);
 600             fjz2             = _mm_macc_ps(dz32,fscal,fjz2);
 601
 602             /**************************
 603              * CALCULATE INTERACTIONS *
 604              **************************/
 605
 606             r33              = _mm_mul_ps(rsq33,rinv33);
 607
 608             /* EWALD ELECTROSTATICS */
 609
 610             /* Analytical PME correction */
 611             zeta2            = _mm_mul_ps(beta2,rsq33);
 612             rinv3            = _mm_mul_ps(rinvsq33,rinv33);
 613             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 614             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 615             felec            = _mm_mul_ps(qq33,felec);
 616             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 617             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv33);
 618             velec            = _mm_mul_ps(qq33,velec);
 619
 620             /* Update potential sum for this i atom from the interaction with this j atom. */
 621             velecsum         = _mm_add_ps(velecsum,velec);
 622
 623             fscal            = felec;
 624
 625              /* Update vectorial force */
 626             fix3             = _mm_macc_ps(dx33,fscal,fix3);
 627             fiy3             = _mm_macc_ps(dy33,fscal,fiy3);
 628             fiz3             = _mm_macc_ps(dz33,fscal,fiz3);
 629
 630             fjx3             = _mm_macc_ps(dx33,fscal,fjx3);
 631             fjy3             = _mm_macc_ps(dy33,fscal,fjy3);
 632             fjz3             = _mm_macc_ps(dz33,fscal,fjz3);
 633
 634             fjptrA             = f+j_coord_offsetA;
 635             fjptrB             = f+j_coord_offsetB;
 636             fjptrC             = f+j_coord_offsetC;
 637             fjptrD             = f+j_coord_offsetD;
 638
 639             gmx_mm_decrement_4rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 640                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
 641                                                    fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
 642
 643             /* Inner loop uses 299 flops */
 644         }
 645
 646         if(jidx<j_index_end)
 647         {
 648
 649             /* Get j neighbor index, and coordinate index */
 650             jnrlistA         = jjnr[jidx];
 651             jnrlistB         = jjnr[jidx+1];
 652             jnrlistC         = jjnr[jidx+2];
 653             jnrlistD         = jjnr[jidx+3];
 654             /* Sign of each element will be negative for non-real atoms.
 655              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 656              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 657              */
 658             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 659             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 660             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 661             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 662             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 663             j_coord_offsetA  = DIM*jnrA;
 664             j_coord_offsetB  = DIM*jnrB;
 665             j_coord_offsetC  = DIM*jnrC;
 666             j_coord_offsetD  = DIM*jnrD;
 667
 668             /* load j atom coordinates */
 669             gmx_mm_load_4rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 670                                               x+j_coord_offsetC,x+j_coord_offsetD,
 671                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
 672                                               &jy2,&jz2,&jx3,&jy3,&jz3);
 673
 674             /* Calculate displacement vector */
 675             dx00             = _mm_sub_ps(ix0,jx0);
 676             dy00             = _mm_sub_ps(iy0,jy0);
 677             dz00             = _mm_sub_ps(iz0,jz0);
 678             dx11             = _mm_sub_ps(ix1,jx1);
 679             dy11             = _mm_sub_ps(iy1,jy1);
 680             dz11             = _mm_sub_ps(iz1,jz1);
 681             dx12             = _mm_sub_ps(ix1,jx2);
 682             dy12             = _mm_sub_ps(iy1,jy2);
 683             dz12             = _mm_sub_ps(iz1,jz2);
 684             dx13             = _mm_sub_ps(ix1,jx3);
 685             dy13             = _mm_sub_ps(iy1,jy3);
 686             dz13             = _mm_sub_ps(iz1,jz3);
 687             dx21             = _mm_sub_ps(ix2,jx1);
 688             dy21             = _mm_sub_ps(iy2,jy1);
 689             dz21             = _mm_sub_ps(iz2,jz1);
 690             dx22             = _mm_sub_ps(ix2,jx2);
 691             dy22             = _mm_sub_ps(iy2,jy2);
 692             dz22             = _mm_sub_ps(iz2,jz2);
 693             dx23             = _mm_sub_ps(ix2,jx3);
 694             dy23             = _mm_sub_ps(iy2,jy3);
 695             dz23             = _mm_sub_ps(iz2,jz3);
 696             dx31             = _mm_sub_ps(ix3,jx1);
 697             dy31             = _mm_sub_ps(iy3,jy1);
 698             dz31             = _mm_sub_ps(iz3,jz1);
 699             dx32             = _mm_sub_ps(ix3,jx2);
 700             dy32             = _mm_sub_ps(iy3,jy2);
 701             dz32             = _mm_sub_ps(iz3,jz2);
 702             dx33             = _mm_sub_ps(ix3,jx3);
 703             dy33             = _mm_sub_ps(iy3,jy3);
 704             dz33             = _mm_sub_ps(iz3,jz3);
 705
 706             /* Calculate squared distance and things based on it */
 707             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 708             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 709             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 710             rsq13            = gmx_mm_calc_rsq_ps(dx13,dy13,dz13);
 711             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 712             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 713             rsq23            = gmx_mm_calc_rsq_ps(dx23,dy23,dz23);
 714             rsq31            = gmx_mm_calc_rsq_ps(dx31,dy31,dz31);
 715             rsq32            = gmx_mm_calc_rsq_ps(dx32,dy32,dz32);
 716             rsq33            = gmx_mm_calc_rsq_ps(dx33,dy33,dz33);
 717
 718             rinv11           = avx128fma_invsqrt_f(rsq11);
 719             rinv12           = avx128fma_invsqrt_f(rsq12);
 720             rinv13           = avx128fma_invsqrt_f(rsq13);
 721             rinv21           = avx128fma_invsqrt_f(rsq21);
 722             rinv22           = avx128fma_invsqrt_f(rsq22);
 723             rinv23           = avx128fma_invsqrt_f(rsq23);
 724             rinv31           = avx128fma_invsqrt_f(rsq31);
 725             rinv32           = avx128fma_invsqrt_f(rsq32);
 726             rinv33           = avx128fma_invsqrt_f(rsq33);
 727
 728             rinvsq00         = avx128fma_inv_f(rsq00);
 729             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 730             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 731             rinvsq13         = _mm_mul_ps(rinv13,rinv13);
 732             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 733             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 734             rinvsq23         = _mm_mul_ps(rinv23,rinv23);
 735             rinvsq31         = _mm_mul_ps(rinv31,rinv31);
 736             rinvsq32         = _mm_mul_ps(rinv32,rinv32);
 737             rinvsq33         = _mm_mul_ps(rinv33,rinv33);
 738
 739             fjx0             = _mm_setzero_ps();
 740             fjy0             = _mm_setzero_ps();
 741             fjz0             = _mm_setzero_ps();
 742             fjx1             = _mm_setzero_ps();
 743             fjy1             = _mm_setzero_ps();
 744             fjz1             = _mm_setzero_ps();
 745             fjx2             = _mm_setzero_ps();
 746             fjy2             = _mm_setzero_ps();
 747             fjz2             = _mm_setzero_ps();
 748             fjx3             = _mm_setzero_ps();
 749             fjy3             = _mm_setzero_ps();
 750             fjz3             = _mm_setzero_ps();
 751
 752             /**************************
 753              * CALCULATE INTERACTIONS *
 754              **************************/
 755
 756             /* LENNARD-JONES DISPERSION/REPULSION */
 757
 758             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 759             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 760             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 761             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 762             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 763
 764             /* Update potential sum for this i atom from the interaction with this j atom. */
 765             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 766             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 767
 768             fscal            = fvdw;
 769
 770             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 771
 772              /* Update vectorial force */
 773             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 774             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 775             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 776
 777             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 778             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 779             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 780
 781             /**************************
 782              * CALCULATE INTERACTIONS *
 783              **************************/
 784
 785             r11              = _mm_mul_ps(rsq11,rinv11);
 786             r11              = _mm_andnot_ps(dummy_mask,r11);
 787
 788             /* EWALD ELECTROSTATICS */
 789
 790             /* Analytical PME correction */
 791             zeta2            = _mm_mul_ps(beta2,rsq11);
 792             rinv3            = _mm_mul_ps(rinvsq11,rinv11);
 793             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 794             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 795             felec            = _mm_mul_ps(qq11,felec);
 796             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 797             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv11);
 798             velec            = _mm_mul_ps(qq11,velec);
 799
 800             /* Update potential sum for this i atom from the interaction with this j atom. */
 801             velec            = _mm_andnot_ps(dummy_mask,velec);
 802             velecsum         = _mm_add_ps(velecsum,velec);
 803
 804             fscal            = felec;
 805
 806             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 807
 808              /* Update vectorial force */
 809             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 810             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 811             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 812
 813             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 814             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 815             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 816
 817             /**************************
 818              * CALCULATE INTERACTIONS *
 819              **************************/
 820
 821             r12              = _mm_mul_ps(rsq12,rinv12);
 822             r12              = _mm_andnot_ps(dummy_mask,r12);
 823
 824             /* EWALD ELECTROSTATICS */
 825
 826             /* Analytical PME correction */
 827             zeta2            = _mm_mul_ps(beta2,rsq12);
 828             rinv3            = _mm_mul_ps(rinvsq12,rinv12);
 829             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 830             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 831             felec            = _mm_mul_ps(qq12,felec);
 832             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 833             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv12);
 834             velec            = _mm_mul_ps(qq12,velec);
 835
 836             /* Update potential sum for this i atom from the interaction with this j atom. */
 837             velec            = _mm_andnot_ps(dummy_mask,velec);
 838             velecsum         = _mm_add_ps(velecsum,velec);
 839
 840             fscal            = felec;
 841
 842             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 843
 844              /* Update vectorial force */
 845             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 846             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 847             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 848
 849             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 850             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 851             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 852
 853             /**************************
 854              * CALCULATE INTERACTIONS *
 855              **************************/
 856
 857             r13              = _mm_mul_ps(rsq13,rinv13);
 858             r13              = _mm_andnot_ps(dummy_mask,r13);
 859
 860             /* EWALD ELECTROSTATICS */
 861
 862             /* Analytical PME correction */
 863             zeta2            = _mm_mul_ps(beta2,rsq13);
 864             rinv3            = _mm_mul_ps(rinvsq13,rinv13);
 865             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 866             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 867             felec            = _mm_mul_ps(qq13,felec);
 868             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 869             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv13);
 870             velec            = _mm_mul_ps(qq13,velec);
 871
 872             /* Update potential sum for this i atom from the interaction with this j atom. */
 873             velec            = _mm_andnot_ps(dummy_mask,velec);
 874             velecsum         = _mm_add_ps(velecsum,velec);
 875
 876             fscal            = felec;
 877
 878             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 879
 880              /* Update vectorial force */
 881             fix1             = _mm_macc_ps(dx13,fscal,fix1);
 882             fiy1             = _mm_macc_ps(dy13,fscal,fiy1);
 883             fiz1             = _mm_macc_ps(dz13,fscal,fiz1);
 884
 885             fjx3             = _mm_macc_ps(dx13,fscal,fjx3);
 886             fjy3             = _mm_macc_ps(dy13,fscal,fjy3);
 887             fjz3             = _mm_macc_ps(dz13,fscal,fjz3);
 888
 889             /**************************
 890              * CALCULATE INTERACTIONS *
 891              **************************/
 892
 893             r21              = _mm_mul_ps(rsq21,rinv21);
 894             r21              = _mm_andnot_ps(dummy_mask,r21);
 895
 896             /* EWALD ELECTROSTATICS */
 897
 898             /* Analytical PME correction */
 899             zeta2            = _mm_mul_ps(beta2,rsq21);
 900             rinv3            = _mm_mul_ps(rinvsq21,rinv21);
 901             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 902             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 903             felec            = _mm_mul_ps(qq21,felec);
 904             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 905             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv21);
 906             velec            = _mm_mul_ps(qq21,velec);
 907
 908             /* Update potential sum for this i atom from the interaction with this j atom. */
 909             velec            = _mm_andnot_ps(dummy_mask,velec);
 910             velecsum         = _mm_add_ps(velecsum,velec);
 911
 912             fscal            = felec;
 913
 914             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 915
 916              /* Update vectorial force */
 917             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 918             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 919             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 920
 921             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 922             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 923             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 924
 925             /**************************
 926              * CALCULATE INTERACTIONS *
 927              **************************/
 928
 929             r22              = _mm_mul_ps(rsq22,rinv22);
 930             r22              = _mm_andnot_ps(dummy_mask,r22);
 931
 932             /* EWALD ELECTROSTATICS */
 933
 934             /* Analytical PME correction */
 935             zeta2            = _mm_mul_ps(beta2,rsq22);
 936             rinv3            = _mm_mul_ps(rinvsq22,rinv22);
 937             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 938             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 939             felec            = _mm_mul_ps(qq22,felec);
 940             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 941             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv22);
 942             velec            = _mm_mul_ps(qq22,velec);
 943
 944             /* Update potential sum for this i atom from the interaction with this j atom. */
 945             velec            = _mm_andnot_ps(dummy_mask,velec);
 946             velecsum         = _mm_add_ps(velecsum,velec);
 947
 948             fscal            = felec;
 949
 950             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 951
 952              /* Update vectorial force */
 953             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 954             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 955             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 956
 957             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 958             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 959             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 960
 961             /**************************
 962              * CALCULATE INTERACTIONS *
 963              **************************/
 964
 965             r23              = _mm_mul_ps(rsq23,rinv23);
 966             r23              = _mm_andnot_ps(dummy_mask,r23);
 967
 968             /* EWALD ELECTROSTATICS */
 969
 970             /* Analytical PME correction */
 971             zeta2            = _mm_mul_ps(beta2,rsq23);
 972             rinv3            = _mm_mul_ps(rinvsq23,rinv23);
 973             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
 974             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 975             felec            = _mm_mul_ps(qq23,felec);
 976             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
 977             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv23);
 978             velec            = _mm_mul_ps(qq23,velec);
 979
 980             /* Update potential sum for this i atom from the interaction with this j atom. */
 981             velec            = _mm_andnot_ps(dummy_mask,velec);
 982             velecsum         = _mm_add_ps(velecsum,velec);
 983
 984             fscal            = felec;
 985
 986             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 987
 988              /* Update vectorial force */
 989             fix2             = _mm_macc_ps(dx23,fscal,fix2);
 990             fiy2             = _mm_macc_ps(dy23,fscal,fiy2);
 991             fiz2             = _mm_macc_ps(dz23,fscal,fiz2);
 992
 993             fjx3             = _mm_macc_ps(dx23,fscal,fjx3);
 994             fjy3             = _mm_macc_ps(dy23,fscal,fjy3);
 995             fjz3             = _mm_macc_ps(dz23,fscal,fjz3);
 996
 997             /**************************
 998              * CALCULATE INTERACTIONS *
 999              **************************/
1000
1001             r31              = _mm_mul_ps(rsq31,rinv31);
1002             r31              = _mm_andnot_ps(dummy_mask,r31);
1003
1004             /* EWALD ELECTROSTATICS */
1005
1006             /* Analytical PME correction */
1007             zeta2            = _mm_mul_ps(beta2,rsq31);
1008             rinv3            = _mm_mul_ps(rinvsq31,rinv31);
1009             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1010             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1011             felec            = _mm_mul_ps(qq31,felec);
1012             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
1013             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv31);
1014             velec            = _mm_mul_ps(qq31,velec);
1015
1016             /* Update potential sum for this i atom from the interaction with this j atom. */
1017             velec            = _mm_andnot_ps(dummy_mask,velec);
1018             velecsum         = _mm_add_ps(velecsum,velec);
1019
1020             fscal            = felec;
1021
1022             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1023
1024              /* Update vectorial force */
1025             fix3             = _mm_macc_ps(dx31,fscal,fix3);
1026             fiy3             = _mm_macc_ps(dy31,fscal,fiy3);
1027             fiz3             = _mm_macc_ps(dz31,fscal,fiz3);
1028
1029             fjx1             = _mm_macc_ps(dx31,fscal,fjx1);
1030             fjy1             = _mm_macc_ps(dy31,fscal,fjy1);
1031             fjz1             = _mm_macc_ps(dz31,fscal,fjz1);
1032
1033             /**************************
1034              * CALCULATE INTERACTIONS *
1035              **************************/
1036
1037             r32              = _mm_mul_ps(rsq32,rinv32);
1038             r32              = _mm_andnot_ps(dummy_mask,r32);
1039
1040             /* EWALD ELECTROSTATICS */
1041
1042             /* Analytical PME correction */
1043             zeta2            = _mm_mul_ps(beta2,rsq32);
1044             rinv3            = _mm_mul_ps(rinvsq32,rinv32);
1045             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1046             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1047             felec            = _mm_mul_ps(qq32,felec);
1048             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
1049             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv32);
1050             velec            = _mm_mul_ps(qq32,velec);
1051
1052             /* Update potential sum for this i atom from the interaction with this j atom. */
1053             velec            = _mm_andnot_ps(dummy_mask,velec);
1054             velecsum         = _mm_add_ps(velecsum,velec);
1055
1056             fscal            = felec;
1057
1058             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1059
1060              /* Update vectorial force */
1061             fix3             = _mm_macc_ps(dx32,fscal,fix3);
1062             fiy3             = _mm_macc_ps(dy32,fscal,fiy3);
1063             fiz3             = _mm_macc_ps(dz32,fscal,fiz3);
1064
1065             fjx2             = _mm_macc_ps(dx32,fscal,fjx2);
1066             fjy2             = _mm_macc_ps(dy32,fscal,fjy2);
1067             fjz2             = _mm_macc_ps(dz32,fscal,fjz2);
1068
1069             /**************************
1070              * CALCULATE INTERACTIONS *
1071              **************************/
1072
1073             r33              = _mm_mul_ps(rsq33,rinv33);
1074             r33              = _mm_andnot_ps(dummy_mask,r33);
1075
1076             /* EWALD ELECTROSTATICS */
1077
1078             /* Analytical PME correction */
1079             zeta2            = _mm_mul_ps(beta2,rsq33);
1080             rinv3            = _mm_mul_ps(rinvsq33,rinv33);
1081             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1082             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1083             felec            = _mm_mul_ps(qq33,felec);
1084             pmecorrV         = avx128fma_pmecorrV_f(zeta2);
1085             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv33);
1086             velec            = _mm_mul_ps(qq33,velec);
1087
1088             /* Update potential sum for this i atom from the interaction with this j atom. */
1089             velec            = _mm_andnot_ps(dummy_mask,velec);
1090             velecsum         = _mm_add_ps(velecsum,velec);
1091
1092             fscal            = felec;
1093
1094             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1095
1096              /* Update vectorial force */
1097             fix3             = _mm_macc_ps(dx33,fscal,fix3);
1098             fiy3             = _mm_macc_ps(dy33,fscal,fiy3);
1099             fiz3             = _mm_macc_ps(dz33,fscal,fiz3);
1100
1101             fjx3             = _mm_macc_ps(dx33,fscal,fjx3);
1102             fjy3             = _mm_macc_ps(dy33,fscal,fjy3);
1103             fjz3             = _mm_macc_ps(dz33,fscal,fjz3);
1104
1105             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1106             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1107             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1108             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1109
1110             gmx_mm_decrement_4rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1111                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
1112                                                    fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1113
1114             /* Inner loop uses 308 flops */
1115         }
1116
1117         /* End of innermost loop */
1118
1119         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1120                                               f+i_coord_offset,fshift+i_shift_offset);
1121
1122         ggid                        = gid[iidx];
1123         /* Update potential energies */
1124         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1125         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1126
1127         /* Increment number of inner iterations */
1128         inneriter                  += j_index_end - j_index_start;
1129
1130         /* Outer loop uses 26 flops */
1131     }
1132
1133     /* Increment number of outer iterations */
1134     outeriter        += nri;
1135
1136     /* Update outer/inner flops */
1137
1138     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_VF,outeriter*26 + inneriter*308);
1139 }
1140 /*
1141  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4W4_F_avx_128_fma_single
1142  * Electrostatics interaction: Ewald
1143  * VdW interaction:            LennardJones
1144  * Geometry:                   Water4-Water4
1145  * Calculate force/pot:        Force
1146  */
1147 void
1148 nb_kernel_ElecEw_VdwLJ_GeomW4W4_F_avx_128_fma_single
1149                     (t_nblist                    * gmx_restrict       nlist,
1150                      rvec                        * gmx_restrict          xx,
1151                      rvec                        * gmx_restrict          ff,
1152                      struct t_forcerec           * gmx_restrict          fr,
1153                      t_mdatoms                   * gmx_restrict     mdatoms,
1154                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1155                      t_nrnb                      * gmx_restrict        nrnb)
1156 {
1157     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1158      * just 0 for non-waters.
1159      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1160      * jnr indices corresponding to data put in the four positions in the SIMD register.
1161      */
1162     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
1163     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1164     int              jnrA,jnrB,jnrC,jnrD;
1165     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1166     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1167     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
1168     real             rcutoff_scalar;
1169     real             *shiftvec,*fshift,*x,*f;
1170     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1171     real             scratch[4*DIM];
1172     __m128           fscal,rcutoff,rcutoff2,jidxall;
1173     int              vdwioffset0;
1174     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1175     int              vdwioffset1;
1176     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1177     int              vdwioffset2;
1178     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1179     int              vdwioffset3;
1180     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
1181     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1182     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1183     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1184     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1185     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1186     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1187     int              vdwjidx3A,vdwjidx3B,vdwjidx3C,vdwjidx3D;
1188     __m128           jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
1189     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1190     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1191     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1192     __m128           dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
1193     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1194     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1195     __m128           dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
1196     __m128           dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
1197     __m128           dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
1198     __m128           dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
1199     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
1200     real             *charge;
1201     int              nvdwtype;
1202     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1203     int              *vdwtype;
1204     real             *vdwparam;
1205     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
1206     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
1207     __m128i          ewitab;
1208     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
1209     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
1210     real             *ewtab;
1211     __m128           dummy_mask,cutoff_mask;
1212     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1213     __m128           one     = _mm_set1_ps(1.0);
1214     __m128           two     = _mm_set1_ps(2.0);
1215     x                = xx[0];
1216     f                = ff[0];
1217
1218     nri              = nlist->nri;
1219     iinr             = nlist->iinr;
1220     jindex           = nlist->jindex;
1221     jjnr             = nlist->jjnr;
1222     shiftidx         = nlist->shift;
1223     gid              = nlist->gid;
1224     shiftvec         = fr->shift_vec[0];
1225     fshift           = fr->fshift[0];
1226     facel            = _mm_set1_ps(fr->ic->epsfac);
1227     charge           = mdatoms->chargeA;
1228     nvdwtype         = fr->ntype;
1229     vdwparam         = fr->nbfp;
1230     vdwtype          = mdatoms->typeA;
1231
1232     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
1233     beta             = _mm_set1_ps(fr->ic->ewaldcoeff_q);
1234     beta2            = _mm_mul_ps(beta,beta);
1235     beta3            = _mm_mul_ps(beta,beta2);
1236     ewtab            = fr->ic->tabq_coul_F;
1237     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
1238     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
1239
1240     /* Setup water-specific parameters */
1241     inr              = nlist->iinr[0];
1242     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1243     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1244     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
1245     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
1246
1247     jq1              = _mm_set1_ps(charge[inr+1]);
1248     jq2              = _mm_set1_ps(charge[inr+2]);
1249     jq3              = _mm_set1_ps(charge[inr+3]);
1250     vdwjidx0A        = 2*vdwtype[inr+0];
1251     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1252     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1253     qq11             = _mm_mul_ps(iq1,jq1);
1254     qq12             = _mm_mul_ps(iq1,jq2);
1255     qq13             = _mm_mul_ps(iq1,jq3);
1256     qq21             = _mm_mul_ps(iq2,jq1);
1257     qq22             = _mm_mul_ps(iq2,jq2);
1258     qq23             = _mm_mul_ps(iq2,jq3);
1259     qq31             = _mm_mul_ps(iq3,jq1);
1260     qq32             = _mm_mul_ps(iq3,jq2);
1261     qq33             = _mm_mul_ps(iq3,jq3);
1262
1263     /* Avoid stupid compiler warnings */
1264     jnrA = jnrB = jnrC = jnrD = 0;
1265     j_coord_offsetA = 0;
1266     j_coord_offsetB = 0;
1267     j_coord_offsetC = 0;
1268     j_coord_offsetD = 0;
1269
1270     outeriter        = 0;
1271     inneriter        = 0;
1272
1273     for(iidx=0;iidx<4*DIM;iidx++)
1274     {
1275         scratch[iidx] = 0.0;
1276     }
1277
1278     /* Start outer loop over neighborlists */
1279     for(iidx=0; iidx<nri; iidx++)
1280     {
1281         /* Load shift vector for this list */
1282         i_shift_offset   = DIM*shiftidx[iidx];
1283
1284         /* Load limits for loop over neighbors */
1285         j_index_start    = jindex[iidx];
1286         j_index_end      = jindex[iidx+1];
1287
1288         /* Get outer coordinate index */
1289         inr              = iinr[iidx];
1290         i_coord_offset   = DIM*inr;
1291
1292         /* Load i particle coords and add shift vector */
1293         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1294                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
1295
1296         fix0             = _mm_setzero_ps();
1297         fiy0             = _mm_setzero_ps();
1298         fiz0             = _mm_setzero_ps();
1299         fix1             = _mm_setzero_ps();
1300         fiy1             = _mm_setzero_ps();
1301         fiz1             = _mm_setzero_ps();
1302         fix2             = _mm_setzero_ps();
1303         fiy2             = _mm_setzero_ps();
1304         fiz2             = _mm_setzero_ps();
1305         fix3             = _mm_setzero_ps();
1306         fiy3             = _mm_setzero_ps();
1307         fiz3             = _mm_setzero_ps();
1308
1309         /* Start inner kernel loop */
1310         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1311         {
1312
1313             /* Get j neighbor index, and coordinate index */
1314             jnrA             = jjnr[jidx];
1315             jnrB             = jjnr[jidx+1];
1316             jnrC             = jjnr[jidx+2];
1317             jnrD             = jjnr[jidx+3];
1318             j_coord_offsetA  = DIM*jnrA;
1319             j_coord_offsetB  = DIM*jnrB;
1320             j_coord_offsetC  = DIM*jnrC;
1321             j_coord_offsetD  = DIM*jnrD;
1322
1323             /* load j atom coordinates */
1324             gmx_mm_load_4rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1325                                               x+j_coord_offsetC,x+j_coord_offsetD,
1326                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1327                                               &jy2,&jz2,&jx3,&jy3,&jz3);
1328
1329             /* Calculate displacement vector */
1330             dx00             = _mm_sub_ps(ix0,jx0);
1331             dy00             = _mm_sub_ps(iy0,jy0);
1332             dz00             = _mm_sub_ps(iz0,jz0);
1333             dx11             = _mm_sub_ps(ix1,jx1);
1334             dy11             = _mm_sub_ps(iy1,jy1);
1335             dz11             = _mm_sub_ps(iz1,jz1);
1336             dx12             = _mm_sub_ps(ix1,jx2);
1337             dy12             = _mm_sub_ps(iy1,jy2);
1338             dz12             = _mm_sub_ps(iz1,jz2);
1339             dx13             = _mm_sub_ps(ix1,jx3);
1340             dy13             = _mm_sub_ps(iy1,jy3);
1341             dz13             = _mm_sub_ps(iz1,jz3);
1342             dx21             = _mm_sub_ps(ix2,jx1);
1343             dy21             = _mm_sub_ps(iy2,jy1);
1344             dz21             = _mm_sub_ps(iz2,jz1);
1345             dx22             = _mm_sub_ps(ix2,jx2);
1346             dy22             = _mm_sub_ps(iy2,jy2);
1347             dz22             = _mm_sub_ps(iz2,jz2);
1348             dx23             = _mm_sub_ps(ix2,jx3);
1349             dy23             = _mm_sub_ps(iy2,jy3);
1350             dz23             = _mm_sub_ps(iz2,jz3);
1351             dx31             = _mm_sub_ps(ix3,jx1);
1352             dy31             = _mm_sub_ps(iy3,jy1);
1353             dz31             = _mm_sub_ps(iz3,jz1);
1354             dx32             = _mm_sub_ps(ix3,jx2);
1355             dy32             = _mm_sub_ps(iy3,jy2);
1356             dz32             = _mm_sub_ps(iz3,jz2);
1357             dx33             = _mm_sub_ps(ix3,jx3);
1358             dy33             = _mm_sub_ps(iy3,jy3);
1359             dz33             = _mm_sub_ps(iz3,jz3);
1360
1361             /* Calculate squared distance and things based on it */
1362             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1363             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1364             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1365             rsq13            = gmx_mm_calc_rsq_ps(dx13,dy13,dz13);
1366             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1367             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1368             rsq23            = gmx_mm_calc_rsq_ps(dx23,dy23,dz23);
1369             rsq31            = gmx_mm_calc_rsq_ps(dx31,dy31,dz31);
1370             rsq32            = gmx_mm_calc_rsq_ps(dx32,dy32,dz32);
1371             rsq33            = gmx_mm_calc_rsq_ps(dx33,dy33,dz33);
1372
1373             rinv11           = avx128fma_invsqrt_f(rsq11);
1374             rinv12           = avx128fma_invsqrt_f(rsq12);
1375             rinv13           = avx128fma_invsqrt_f(rsq13);
1376             rinv21           = avx128fma_invsqrt_f(rsq21);
1377             rinv22           = avx128fma_invsqrt_f(rsq22);
1378             rinv23           = avx128fma_invsqrt_f(rsq23);
1379             rinv31           = avx128fma_invsqrt_f(rsq31);
1380             rinv32           = avx128fma_invsqrt_f(rsq32);
1381             rinv33           = avx128fma_invsqrt_f(rsq33);
1382
1383             rinvsq00         = avx128fma_inv_f(rsq00);
1384             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1385             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1386             rinvsq13         = _mm_mul_ps(rinv13,rinv13);
1387             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1388             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1389             rinvsq23         = _mm_mul_ps(rinv23,rinv23);
1390             rinvsq31         = _mm_mul_ps(rinv31,rinv31);
1391             rinvsq32         = _mm_mul_ps(rinv32,rinv32);
1392             rinvsq33         = _mm_mul_ps(rinv33,rinv33);
1393
1394             fjx0             = _mm_setzero_ps();
1395             fjy0             = _mm_setzero_ps();
1396             fjz0             = _mm_setzero_ps();
1397             fjx1             = _mm_setzero_ps();
1398             fjy1             = _mm_setzero_ps();
1399             fjz1             = _mm_setzero_ps();
1400             fjx2             = _mm_setzero_ps();
1401             fjy2             = _mm_setzero_ps();
1402             fjz2             = _mm_setzero_ps();
1403             fjx3             = _mm_setzero_ps();
1404             fjy3             = _mm_setzero_ps();
1405             fjz3             = _mm_setzero_ps();
1406
1407             /**************************
1408              * CALCULATE INTERACTIONS *
1409              **************************/
1410
1411             /* LENNARD-JONES DISPERSION/REPULSION */
1412
1413             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1414             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1415
1416             fscal            = fvdw;
1417
1418              /* Update vectorial force */
1419             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1420             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1421             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1422
1423             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1424             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1425             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1426
1427             /**************************
1428              * CALCULATE INTERACTIONS *
1429              **************************/
1430
1431             r11              = _mm_mul_ps(rsq11,rinv11);
1432
1433             /* EWALD ELECTROSTATICS */
1434
1435             /* Analytical PME correction */
1436             zeta2            = _mm_mul_ps(beta2,rsq11);
1437             rinv3            = _mm_mul_ps(rinvsq11,rinv11);
1438             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1439             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1440             felec            = _mm_mul_ps(qq11,felec);
1441
1442             fscal            = felec;
1443
1444              /* Update vectorial force */
1445             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1446             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1447             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1448
1449             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1450             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1451             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1452
1453             /**************************
1454              * CALCULATE INTERACTIONS *
1455              **************************/
1456
1457             r12              = _mm_mul_ps(rsq12,rinv12);
1458
1459             /* EWALD ELECTROSTATICS */
1460
1461             /* Analytical PME correction */
1462             zeta2            = _mm_mul_ps(beta2,rsq12);
1463             rinv3            = _mm_mul_ps(rinvsq12,rinv12);
1464             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1465             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1466             felec            = _mm_mul_ps(qq12,felec);
1467
1468             fscal            = felec;
1469
1470              /* Update vectorial force */
1471             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1472             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1473             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1474
1475             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1476             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1477             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1478
1479             /**************************
1480              * CALCULATE INTERACTIONS *
1481              **************************/
1482
1483             r13              = _mm_mul_ps(rsq13,rinv13);
1484
1485             /* EWALD ELECTROSTATICS */
1486
1487             /* Analytical PME correction */
1488             zeta2            = _mm_mul_ps(beta2,rsq13);
1489             rinv3            = _mm_mul_ps(rinvsq13,rinv13);
1490             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1491             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1492             felec            = _mm_mul_ps(qq13,felec);
1493
1494             fscal            = felec;
1495
1496              /* Update vectorial force */
1497             fix1             = _mm_macc_ps(dx13,fscal,fix1);
1498             fiy1             = _mm_macc_ps(dy13,fscal,fiy1);
1499             fiz1             = _mm_macc_ps(dz13,fscal,fiz1);
1500
1501             fjx3             = _mm_macc_ps(dx13,fscal,fjx3);
1502             fjy3             = _mm_macc_ps(dy13,fscal,fjy3);
1503             fjz3             = _mm_macc_ps(dz13,fscal,fjz3);
1504
1505             /**************************
1506              * CALCULATE INTERACTIONS *
1507              **************************/
1508
1509             r21              = _mm_mul_ps(rsq21,rinv21);
1510
1511             /* EWALD ELECTROSTATICS */
1512
1513             /* Analytical PME correction */
1514             zeta2            = _mm_mul_ps(beta2,rsq21);
1515             rinv3            = _mm_mul_ps(rinvsq21,rinv21);
1516             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1517             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1518             felec            = _mm_mul_ps(qq21,felec);
1519
1520             fscal            = felec;
1521
1522              /* Update vectorial force */
1523             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1524             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1525             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1526
1527             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1528             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1529             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1530
1531             /**************************
1532              * CALCULATE INTERACTIONS *
1533              **************************/
1534
1535             r22              = _mm_mul_ps(rsq22,rinv22);
1536
1537             /* EWALD ELECTROSTATICS */
1538
1539             /* Analytical PME correction */
1540             zeta2            = _mm_mul_ps(beta2,rsq22);
1541             rinv3            = _mm_mul_ps(rinvsq22,rinv22);
1542             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1543             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1544             felec            = _mm_mul_ps(qq22,felec);
1545
1546             fscal            = felec;
1547
1548              /* Update vectorial force */
1549             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1550             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1551             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1552
1553             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1554             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1555             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1556
1557             /**************************
1558              * CALCULATE INTERACTIONS *
1559              **************************/
1560
1561             r23              = _mm_mul_ps(rsq23,rinv23);
1562
1563             /* EWALD ELECTROSTATICS */
1564
1565             /* Analytical PME correction */
1566             zeta2            = _mm_mul_ps(beta2,rsq23);
1567             rinv3            = _mm_mul_ps(rinvsq23,rinv23);
1568             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1569             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1570             felec            = _mm_mul_ps(qq23,felec);
1571
1572             fscal            = felec;
1573
1574              /* Update vectorial force */
1575             fix2             = _mm_macc_ps(dx23,fscal,fix2);
1576             fiy2             = _mm_macc_ps(dy23,fscal,fiy2);
1577             fiz2             = _mm_macc_ps(dz23,fscal,fiz2);
1578
1579             fjx3             = _mm_macc_ps(dx23,fscal,fjx3);
1580             fjy3             = _mm_macc_ps(dy23,fscal,fjy3);
1581             fjz3             = _mm_macc_ps(dz23,fscal,fjz3);
1582
1583             /**************************
1584              * CALCULATE INTERACTIONS *
1585              **************************/
1586
1587             r31              = _mm_mul_ps(rsq31,rinv31);
1588
1589             /* EWALD ELECTROSTATICS */
1590
1591             /* Analytical PME correction */
1592             zeta2            = _mm_mul_ps(beta2,rsq31);
1593             rinv3            = _mm_mul_ps(rinvsq31,rinv31);
1594             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1595             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1596             felec            = _mm_mul_ps(qq31,felec);
1597
1598             fscal            = felec;
1599
1600              /* Update vectorial force */
1601             fix3             = _mm_macc_ps(dx31,fscal,fix3);
1602             fiy3             = _mm_macc_ps(dy31,fscal,fiy3);
1603             fiz3             = _mm_macc_ps(dz31,fscal,fiz3);
1604
1605             fjx1             = _mm_macc_ps(dx31,fscal,fjx1);
1606             fjy1             = _mm_macc_ps(dy31,fscal,fjy1);
1607             fjz1             = _mm_macc_ps(dz31,fscal,fjz1);
1608
1609             /**************************
1610              * CALCULATE INTERACTIONS *
1611              **************************/
1612
1613             r32              = _mm_mul_ps(rsq32,rinv32);
1614
1615             /* EWALD ELECTROSTATICS */
1616
1617             /* Analytical PME correction */
1618             zeta2            = _mm_mul_ps(beta2,rsq32);
1619             rinv3            = _mm_mul_ps(rinvsq32,rinv32);
1620             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1621             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1622             felec            = _mm_mul_ps(qq32,felec);
1623
1624             fscal            = felec;
1625
1626              /* Update vectorial force */
1627             fix3             = _mm_macc_ps(dx32,fscal,fix3);
1628             fiy3             = _mm_macc_ps(dy32,fscal,fiy3);
1629             fiz3             = _mm_macc_ps(dz32,fscal,fiz3);
1630
1631             fjx2             = _mm_macc_ps(dx32,fscal,fjx2);
1632             fjy2             = _mm_macc_ps(dy32,fscal,fjy2);
1633             fjz2             = _mm_macc_ps(dz32,fscal,fjz2);
1634
1635             /**************************
1636              * CALCULATE INTERACTIONS *
1637              **************************/
1638
1639             r33              = _mm_mul_ps(rsq33,rinv33);
1640
1641             /* EWALD ELECTROSTATICS */
1642
1643             /* Analytical PME correction */
1644             zeta2            = _mm_mul_ps(beta2,rsq33);
1645             rinv3            = _mm_mul_ps(rinvsq33,rinv33);
1646             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1647             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1648             felec            = _mm_mul_ps(qq33,felec);
1649
1650             fscal            = felec;
1651
1652              /* Update vectorial force */
1653             fix3             = _mm_macc_ps(dx33,fscal,fix3);
1654             fiy3             = _mm_macc_ps(dy33,fscal,fiy3);
1655             fiz3             = _mm_macc_ps(dz33,fscal,fiz3);
1656
1657             fjx3             = _mm_macc_ps(dx33,fscal,fjx3);
1658             fjy3             = _mm_macc_ps(dy33,fscal,fjy3);
1659             fjz3             = _mm_macc_ps(dz33,fscal,fjz3);
1660
1661             fjptrA             = f+j_coord_offsetA;
1662             fjptrB             = f+j_coord_offsetB;
1663             fjptrC             = f+j_coord_offsetC;
1664             fjptrD             = f+j_coord_offsetD;
1665
1666             gmx_mm_decrement_4rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1667                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
1668                                                    fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1669
1670             /* Inner loop uses 285 flops */
1671         }
1672
1673         if(jidx<j_index_end)
1674         {
1675
1676             /* Get j neighbor index, and coordinate index */
1677             jnrlistA         = jjnr[jidx];
1678             jnrlistB         = jjnr[jidx+1];
1679             jnrlistC         = jjnr[jidx+2];
1680             jnrlistD         = jjnr[jidx+3];
1681             /* Sign of each element will be negative for non-real atoms.
1682              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1683              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1684              */
1685             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1686             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
1687             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
1688             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1689             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1690             j_coord_offsetA  = DIM*jnrA;
1691             j_coord_offsetB  = DIM*jnrB;
1692             j_coord_offsetC  = DIM*jnrC;
1693             j_coord_offsetD  = DIM*jnrD;
1694
1695             /* load j atom coordinates */
1696             gmx_mm_load_4rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1697                                               x+j_coord_offsetC,x+j_coord_offsetD,
1698                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1699                                               &jy2,&jz2,&jx3,&jy3,&jz3);
1700
1701             /* Calculate displacement vector */
1702             dx00             = _mm_sub_ps(ix0,jx0);
1703             dy00             = _mm_sub_ps(iy0,jy0);
1704             dz00             = _mm_sub_ps(iz0,jz0);
1705             dx11             = _mm_sub_ps(ix1,jx1);
1706             dy11             = _mm_sub_ps(iy1,jy1);
1707             dz11             = _mm_sub_ps(iz1,jz1);
1708             dx12             = _mm_sub_ps(ix1,jx2);
1709             dy12             = _mm_sub_ps(iy1,jy2);
1710             dz12             = _mm_sub_ps(iz1,jz2);
1711             dx13             = _mm_sub_ps(ix1,jx3);
1712             dy13             = _mm_sub_ps(iy1,jy3);
1713             dz13             = _mm_sub_ps(iz1,jz3);
1714             dx21             = _mm_sub_ps(ix2,jx1);
1715             dy21             = _mm_sub_ps(iy2,jy1);
1716             dz21             = _mm_sub_ps(iz2,jz1);
1717             dx22             = _mm_sub_ps(ix2,jx2);
1718             dy22             = _mm_sub_ps(iy2,jy2);
1719             dz22             = _mm_sub_ps(iz2,jz2);
1720             dx23             = _mm_sub_ps(ix2,jx3);
1721             dy23             = _mm_sub_ps(iy2,jy3);
1722             dz23             = _mm_sub_ps(iz2,jz3);
1723             dx31             = _mm_sub_ps(ix3,jx1);
1724             dy31             = _mm_sub_ps(iy3,jy1);
1725             dz31             = _mm_sub_ps(iz3,jz1);
1726             dx32             = _mm_sub_ps(ix3,jx2);
1727             dy32             = _mm_sub_ps(iy3,jy2);
1728             dz32             = _mm_sub_ps(iz3,jz2);
1729             dx33             = _mm_sub_ps(ix3,jx3);
1730             dy33             = _mm_sub_ps(iy3,jy3);
1731             dz33             = _mm_sub_ps(iz3,jz3);
1732
1733             /* Calculate squared distance and things based on it */
1734             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1735             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1736             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1737             rsq13            = gmx_mm_calc_rsq_ps(dx13,dy13,dz13);
1738             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1739             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1740             rsq23            = gmx_mm_calc_rsq_ps(dx23,dy23,dz23);
1741             rsq31            = gmx_mm_calc_rsq_ps(dx31,dy31,dz31);
1742             rsq32            = gmx_mm_calc_rsq_ps(dx32,dy32,dz32);
1743             rsq33            = gmx_mm_calc_rsq_ps(dx33,dy33,dz33);
1744
1745             rinv11           = avx128fma_invsqrt_f(rsq11);
1746             rinv12           = avx128fma_invsqrt_f(rsq12);
1747             rinv13           = avx128fma_invsqrt_f(rsq13);
1748             rinv21           = avx128fma_invsqrt_f(rsq21);
1749             rinv22           = avx128fma_invsqrt_f(rsq22);
1750             rinv23           = avx128fma_invsqrt_f(rsq23);
1751             rinv31           = avx128fma_invsqrt_f(rsq31);
1752             rinv32           = avx128fma_invsqrt_f(rsq32);
1753             rinv33           = avx128fma_invsqrt_f(rsq33);
1754
1755             rinvsq00         = avx128fma_inv_f(rsq00);
1756             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1757             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1758             rinvsq13         = _mm_mul_ps(rinv13,rinv13);
1759             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1760             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1761             rinvsq23         = _mm_mul_ps(rinv23,rinv23);
1762             rinvsq31         = _mm_mul_ps(rinv31,rinv31);
1763             rinvsq32         = _mm_mul_ps(rinv32,rinv32);
1764             rinvsq33         = _mm_mul_ps(rinv33,rinv33);
1765
1766             fjx0             = _mm_setzero_ps();
1767             fjy0             = _mm_setzero_ps();
1768             fjz0             = _mm_setzero_ps();
1769             fjx1             = _mm_setzero_ps();
1770             fjy1             = _mm_setzero_ps();
1771             fjz1             = _mm_setzero_ps();
1772             fjx2             = _mm_setzero_ps();
1773             fjy2             = _mm_setzero_ps();
1774             fjz2             = _mm_setzero_ps();
1775             fjx3             = _mm_setzero_ps();
1776             fjy3             = _mm_setzero_ps();
1777             fjz3             = _mm_setzero_ps();
1778
1779             /**************************
1780              * CALCULATE INTERACTIONS *
1781              **************************/
1782
1783             /* LENNARD-JONES DISPERSION/REPULSION */
1784
1785             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1786             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1787
1788             fscal            = fvdw;
1789
1790             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1791
1792              /* Update vectorial force */
1793             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1794             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1795             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1796
1797             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1798             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1799             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1800
1801             /**************************
1802              * CALCULATE INTERACTIONS *
1803              **************************/
1804
1805             r11              = _mm_mul_ps(rsq11,rinv11);
1806             r11              = _mm_andnot_ps(dummy_mask,r11);
1807
1808             /* EWALD ELECTROSTATICS */
1809
1810             /* Analytical PME correction */
1811             zeta2            = _mm_mul_ps(beta2,rsq11);
1812             rinv3            = _mm_mul_ps(rinvsq11,rinv11);
1813             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1814             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1815             felec            = _mm_mul_ps(qq11,felec);
1816
1817             fscal            = felec;
1818
1819             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1820
1821              /* Update vectorial force */
1822             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1823             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1824             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1825
1826             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1827             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1828             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1829
1830             /**************************
1831              * CALCULATE INTERACTIONS *
1832              **************************/
1833
1834             r12              = _mm_mul_ps(rsq12,rinv12);
1835             r12              = _mm_andnot_ps(dummy_mask,r12);
1836
1837             /* EWALD ELECTROSTATICS */
1838
1839             /* Analytical PME correction */
1840             zeta2            = _mm_mul_ps(beta2,rsq12);
1841             rinv3            = _mm_mul_ps(rinvsq12,rinv12);
1842             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1843             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1844             felec            = _mm_mul_ps(qq12,felec);
1845
1846             fscal            = felec;
1847
1848             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1849
1850              /* Update vectorial force */
1851             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1852             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1853             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1854
1855             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1856             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1857             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1858
1859             /**************************
1860              * CALCULATE INTERACTIONS *
1861              **************************/
1862
1863             r13              = _mm_mul_ps(rsq13,rinv13);
1864             r13              = _mm_andnot_ps(dummy_mask,r13);
1865
1866             /* EWALD ELECTROSTATICS */
1867
1868             /* Analytical PME correction */
1869             zeta2            = _mm_mul_ps(beta2,rsq13);
1870             rinv3            = _mm_mul_ps(rinvsq13,rinv13);
1871             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1872             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1873             felec            = _mm_mul_ps(qq13,felec);
1874
1875             fscal            = felec;
1876
1877             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1878
1879              /* Update vectorial force */
1880             fix1             = _mm_macc_ps(dx13,fscal,fix1);
1881             fiy1             = _mm_macc_ps(dy13,fscal,fiy1);
1882             fiz1             = _mm_macc_ps(dz13,fscal,fiz1);
1883
1884             fjx3             = _mm_macc_ps(dx13,fscal,fjx3);
1885             fjy3             = _mm_macc_ps(dy13,fscal,fjy3);
1886             fjz3             = _mm_macc_ps(dz13,fscal,fjz3);
1887
1888             /**************************
1889              * CALCULATE INTERACTIONS *
1890              **************************/
1891
1892             r21              = _mm_mul_ps(rsq21,rinv21);
1893             r21              = _mm_andnot_ps(dummy_mask,r21);
1894
1895             /* EWALD ELECTROSTATICS */
1896
1897             /* Analytical PME correction */
1898             zeta2            = _mm_mul_ps(beta2,rsq21);
1899             rinv3            = _mm_mul_ps(rinvsq21,rinv21);
1900             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1901             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1902             felec            = _mm_mul_ps(qq21,felec);
1903
1904             fscal            = felec;
1905
1906             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1907
1908              /* Update vectorial force */
1909             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1910             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1911             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1912
1913             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1914             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1915             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1916
1917             /**************************
1918              * CALCULATE INTERACTIONS *
1919              **************************/
1920
1921             r22              = _mm_mul_ps(rsq22,rinv22);
1922             r22              = _mm_andnot_ps(dummy_mask,r22);
1923
1924             /* EWALD ELECTROSTATICS */
1925
1926             /* Analytical PME correction */
1927             zeta2            = _mm_mul_ps(beta2,rsq22);
1928             rinv3            = _mm_mul_ps(rinvsq22,rinv22);
1929             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1930             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1931             felec            = _mm_mul_ps(qq22,felec);
1932
1933             fscal            = felec;
1934
1935             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1936
1937              /* Update vectorial force */
1938             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1939             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1940             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1941
1942             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1943             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1944             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1945
1946             /**************************
1947              * CALCULATE INTERACTIONS *
1948              **************************/
1949
1950             r23              = _mm_mul_ps(rsq23,rinv23);
1951             r23              = _mm_andnot_ps(dummy_mask,r23);
1952
1953             /* EWALD ELECTROSTATICS */
1954
1955             /* Analytical PME correction */
1956             zeta2            = _mm_mul_ps(beta2,rsq23);
1957             rinv3            = _mm_mul_ps(rinvsq23,rinv23);
1958             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1959             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1960             felec            = _mm_mul_ps(qq23,felec);
1961
1962             fscal            = felec;
1963
1964             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1965
1966              /* Update vectorial force */
1967             fix2             = _mm_macc_ps(dx23,fscal,fix2);
1968             fiy2             = _mm_macc_ps(dy23,fscal,fiy2);
1969             fiz2             = _mm_macc_ps(dz23,fscal,fiz2);
1970
1971             fjx3             = _mm_macc_ps(dx23,fscal,fjx3);
1972             fjy3             = _mm_macc_ps(dy23,fscal,fjy3);
1973             fjz3             = _mm_macc_ps(dz23,fscal,fjz3);
1974
1975             /**************************
1976              * CALCULATE INTERACTIONS *
1977              **************************/
1978
1979             r31              = _mm_mul_ps(rsq31,rinv31);
1980             r31              = _mm_andnot_ps(dummy_mask,r31);
1981
1982             /* EWALD ELECTROSTATICS */
1983
1984             /* Analytical PME correction */
1985             zeta2            = _mm_mul_ps(beta2,rsq31);
1986             rinv3            = _mm_mul_ps(rinvsq31,rinv31);
1987             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
1988             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1989             felec            = _mm_mul_ps(qq31,felec);
1990
1991             fscal            = felec;
1992
1993             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1994
1995              /* Update vectorial force */
1996             fix3             = _mm_macc_ps(dx31,fscal,fix3);
1997             fiy3             = _mm_macc_ps(dy31,fscal,fiy3);
1998             fiz3             = _mm_macc_ps(dz31,fscal,fiz3);
1999
2000             fjx1             = _mm_macc_ps(dx31,fscal,fjx1);
2001             fjy1             = _mm_macc_ps(dy31,fscal,fjy1);
2002             fjz1             = _mm_macc_ps(dz31,fscal,fjz1);
2003
2004             /**************************
2005              * CALCULATE INTERACTIONS *
2006              **************************/
2007
2008             r32              = _mm_mul_ps(rsq32,rinv32);
2009             r32              = _mm_andnot_ps(dummy_mask,r32);
2010
2011             /* EWALD ELECTROSTATICS */
2012
2013             /* Analytical PME correction */
2014             zeta2            = _mm_mul_ps(beta2,rsq32);
2015             rinv3            = _mm_mul_ps(rinvsq32,rinv32);
2016             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
2017             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
2018             felec            = _mm_mul_ps(qq32,felec);
2019
2020             fscal            = felec;
2021
2022             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2023
2024              /* Update vectorial force */
2025             fix3             = _mm_macc_ps(dx32,fscal,fix3);
2026             fiy3             = _mm_macc_ps(dy32,fscal,fiy3);
2027             fiz3             = _mm_macc_ps(dz32,fscal,fiz3);
2028
2029             fjx2             = _mm_macc_ps(dx32,fscal,fjx2);
2030             fjy2             = _mm_macc_ps(dy32,fscal,fjy2);
2031             fjz2             = _mm_macc_ps(dz32,fscal,fjz2);
2032
2033             /**************************
2034              * CALCULATE INTERACTIONS *
2035              **************************/
2036
2037             r33              = _mm_mul_ps(rsq33,rinv33);
2038             r33              = _mm_andnot_ps(dummy_mask,r33);
2039
2040             /* EWALD ELECTROSTATICS */
2041
2042             /* Analytical PME correction */
2043             zeta2            = _mm_mul_ps(beta2,rsq33);
2044             rinv3            = _mm_mul_ps(rinvsq33,rinv33);
2045             pmecorrF         = avx128fma_pmecorrF_f(zeta2);
2046             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
2047             felec            = _mm_mul_ps(qq33,felec);
2048
2049             fscal            = felec;
2050
2051             fscal            = _mm_andnot_ps(dummy_mask,fscal);
2052
2053              /* Update vectorial force */
2054             fix3             = _mm_macc_ps(dx33,fscal,fix3);
2055             fiy3             = _mm_macc_ps(dy33,fscal,fiy3);
2056             fiz3             = _mm_macc_ps(dz33,fscal,fiz3);
2057
2058             fjx3             = _mm_macc_ps(dx33,fscal,fjx3);
2059             fjy3             = _mm_macc_ps(dy33,fscal,fjy3);
2060             fjz3             = _mm_macc_ps(dz33,fscal,fjz3);
2061
2062             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
2063             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
2064             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
2065             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
2066
2067             gmx_mm_decrement_4rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
2068                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
2069                                                    fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
2070
2071             /* Inner loop uses 294 flops */
2072         }
2073
2074         /* End of innermost loop */
2075
2076         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
2077                                               f+i_coord_offset,fshift+i_shift_offset);
2078
2079         /* Increment number of inner iterations */
2080         inneriter                  += j_index_end - j_index_start;
2081
2082         /* Outer loop uses 24 flops */
2083     }
2084
2085     /* Increment number of outer iterations */
2086     outeriter        += nri;
2087
2088     /* Update outer/inner flops */
2089
2090     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_F,outeriter*24 + inneriter*294);
2091 }