src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_double/nb_kernel_ElecEwSh_VdwLJEwSh_GeomW4W4_avx_128_fma_double.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014,2015,2017, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_double kernel generator.
  37  */
  38 #include "gmxpre.h"
  39
  40 #include "config.h"
  41
  42 #include <math.h>
  43
  44 #include "../nb_kernel.h"
  45 #include "gromacs/gmxlib/nrnb.h"
  46
  47 #include "kernelutil_x86_avx_128_fma_double.h"
  48
  49 /*
  50  * Gromacs nonbonded kernel:   nb_kernel_ElecEwSh_VdwLJEwSh_GeomW4W4_VF_avx_128_fma_double
  51  * Electrostatics interaction: Ewald
  52  * VdW interaction:            LJEwald
  53  * Geometry:                   Water4-Water4
  54  * Calculate force/pot:        PotentialAndForce
  55  */
  56 void
  57 nb_kernel_ElecEwSh_VdwLJEwSh_GeomW4W4_VF_avx_128_fma_double
  58                     (t_nblist                    * gmx_restrict       nlist,
  59                      rvec                        * gmx_restrict          xx,
  60                      rvec                        * gmx_restrict          ff,
  61                      struct t_forcerec           * gmx_restrict          fr,
  62                      t_mdatoms                   * gmx_restrict     mdatoms,
  63                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  64                      t_nrnb                      * gmx_restrict        nrnb)
  65 {
  66     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  67      * just 0 for non-waters.
  68      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
  69      * jnr indices corresponding to data put in the four positions in the SIMD register.
  70      */
  71     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  72     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  73     int              jnrA,jnrB;
  74     int              j_coord_offsetA,j_coord_offsetB;
  75     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  76     real             rcutoff_scalar;
  77     real             *shiftvec,*fshift,*x,*f;
  78     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
  79     int              vdwioffset0;
  80     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  81     int              vdwioffset1;
  82     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  83     int              vdwioffset2;
  84     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  85     int              vdwioffset3;
  86     __m128d          ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
  87     int              vdwjidx0A,vdwjidx0B;
  88     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  89     int              vdwjidx1A,vdwjidx1B;
  90     __m128d          jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  91     int              vdwjidx2A,vdwjidx2B;
  92     __m128d          jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  93     int              vdwjidx3A,vdwjidx3B;
  94     __m128d          jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
  95     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  96     __m128d          dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
  97     __m128d          dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
  98     __m128d          dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
  99     __m128d          dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 100     __m128d          dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 101     __m128d          dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
 102     __m128d          dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
 103     __m128d          dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
 104     __m128d          dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
 105     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
 106     real             *charge;
 107     int              nvdwtype;
 108     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 109     int              *vdwtype;
 110     real             *vdwparam;
 111     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
 112     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
 113     __m128d           c6grid_00;
 114     __m128d           c6grid_11;
 115     __m128d           c6grid_12;
 116     __m128d           c6grid_13;
 117     __m128d           c6grid_21;
 118     __m128d           c6grid_22;
 119     __m128d           c6grid_23;
 120     __m128d           c6grid_31;
 121     __m128d           c6grid_32;
 122     __m128d           c6grid_33;
 123     real             *vdwgridparam;
 124     __m128d           ewclj,ewclj2,ewclj6,ewcljrsq,poly,exponent,f6A,f6B,sh_lj_ewald;
 125     __m128d           one_half  = _mm_set1_pd(0.5);
 126     __m128d           minus_one = _mm_set1_pd(-1.0);
 127     __m128i          ewitab;
 128     __m128d          ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
 129     real             *ewtab;
 130     __m128d          dummy_mask,cutoff_mask;
 131     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
 132     __m128d          one     = _mm_set1_pd(1.0);
 133     __m128d          two     = _mm_set1_pd(2.0);
 134     x                = xx[0];
 135     f                = ff[0];
 136
 137     nri              = nlist->nri;
 138     iinr             = nlist->iinr;
 139     jindex           = nlist->jindex;
 140     jjnr             = nlist->jjnr;
 141     shiftidx         = nlist->shift;
 142     gid              = nlist->gid;
 143     shiftvec         = fr->shift_vec[0];
 144     fshift           = fr->fshift[0];
 145     facel            = _mm_set1_pd(fr->ic->epsfac);
 146     charge           = mdatoms->chargeA;
 147     nvdwtype         = fr->ntype;
 148     vdwparam         = fr->nbfp;
 149     vdwtype          = mdatoms->typeA;
 150     vdwgridparam     = fr->ljpme_c6grid;
 151     sh_lj_ewald      = _mm_set1_pd(fr->ic->sh_lj_ewald);
 152     ewclj            = _mm_set1_pd(fr->ic->ewaldcoeff_lj);
 153     ewclj2           = _mm_mul_pd(minus_one,_mm_mul_pd(ewclj,ewclj));
 154
 155     sh_ewald         = _mm_set1_pd(fr->ic->sh_ewald);
 156     ewtab            = fr->ic->tabq_coul_FDV0;
 157     ewtabscale       = _mm_set1_pd(fr->ic->tabq_scale);
 158     ewtabhalfspace   = _mm_set1_pd(0.5/fr->ic->tabq_scale);
 159
 160     /* Setup water-specific parameters */
 161     inr              = nlist->iinr[0];
 162     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
 163     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
 164     iq3              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
 165     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 166
 167     jq1              = _mm_set1_pd(charge[inr+1]);
 168     jq2              = _mm_set1_pd(charge[inr+2]);
 169     jq3              = _mm_set1_pd(charge[inr+3]);
 170     vdwjidx0A        = 2*vdwtype[inr+0];
 171     c6_00            = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
 172     c12_00           = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
 173     c6grid_00        = _mm_set1_pd(vdwgridparam[vdwioffset0+vdwjidx0A]);
 174     qq11             = _mm_mul_pd(iq1,jq1);
 175     qq12             = _mm_mul_pd(iq1,jq2);
 176     qq13             = _mm_mul_pd(iq1,jq3);
 177     qq21             = _mm_mul_pd(iq2,jq1);
 178     qq22             = _mm_mul_pd(iq2,jq2);
 179     qq23             = _mm_mul_pd(iq2,jq3);
 180     qq31             = _mm_mul_pd(iq3,jq1);
 181     qq32             = _mm_mul_pd(iq3,jq2);
 182     qq33             = _mm_mul_pd(iq3,jq3);
 183
 184     /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
 185     rcutoff_scalar   = fr->ic->rcoulomb;
 186     rcutoff          = _mm_set1_pd(rcutoff_scalar);
 187     rcutoff2         = _mm_mul_pd(rcutoff,rcutoff);
 188
 189     sh_vdw_invrcut6  = _mm_set1_pd(fr->ic->sh_invrc6);
 190     rvdw             = _mm_set1_pd(fr->ic->rvdw);
 191
 192     /* Avoid stupid compiler warnings */
 193     jnrA = jnrB = 0;
 194     j_coord_offsetA = 0;
 195     j_coord_offsetB = 0;
 196
 197     outeriter        = 0;
 198     inneriter        = 0;
 199
 200     /* Start outer loop over neighborlists */
 201     for(iidx=0; iidx<nri; iidx++)
 202     {
 203         /* Load shift vector for this list */
 204         i_shift_offset   = DIM*shiftidx[iidx];
 205
 206         /* Load limits for loop over neighbors */
 207         j_index_start    = jindex[iidx];
 208         j_index_end      = jindex[iidx+1];
 209
 210         /* Get outer coordinate index */
 211         inr              = iinr[iidx];
 212         i_coord_offset   = DIM*inr;
 213
 214         /* Load i particle coords and add shift vector */
 215         gmx_mm_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
 216                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 217
 218         fix0             = _mm_setzero_pd();
 219         fiy0             = _mm_setzero_pd();
 220         fiz0             = _mm_setzero_pd();
 221         fix1             = _mm_setzero_pd();
 222         fiy1             = _mm_setzero_pd();
 223         fiz1             = _mm_setzero_pd();
 224         fix2             = _mm_setzero_pd();
 225         fiy2             = _mm_setzero_pd();
 226         fiz2             = _mm_setzero_pd();
 227         fix3             = _mm_setzero_pd();
 228         fiy3             = _mm_setzero_pd();
 229         fiz3             = _mm_setzero_pd();
 230
 231         /* Reset potential sums */
 232         velecsum         = _mm_setzero_pd();
 233         vvdwsum          = _mm_setzero_pd();
 234
 235         /* Start inner kernel loop */
 236         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
 237         {
 238
 239             /* Get j neighbor index, and coordinate index */
 240             jnrA             = jjnr[jidx];
 241             jnrB             = jjnr[jidx+1];
 242             j_coord_offsetA  = DIM*jnrA;
 243             j_coord_offsetB  = DIM*jnrB;
 244
 245             /* load j atom coordinates */
 246             gmx_mm_load_4rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
 247                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
 248                                               &jy2,&jz2,&jx3,&jy3,&jz3);
 249
 250             /* Calculate displacement vector */
 251             dx00             = _mm_sub_pd(ix0,jx0);
 252             dy00             = _mm_sub_pd(iy0,jy0);
 253             dz00             = _mm_sub_pd(iz0,jz0);
 254             dx11             = _mm_sub_pd(ix1,jx1);
 255             dy11             = _mm_sub_pd(iy1,jy1);
 256             dz11             = _mm_sub_pd(iz1,jz1);
 257             dx12             = _mm_sub_pd(ix1,jx2);
 258             dy12             = _mm_sub_pd(iy1,jy2);
 259             dz12             = _mm_sub_pd(iz1,jz2);
 260             dx13             = _mm_sub_pd(ix1,jx3);
 261             dy13             = _mm_sub_pd(iy1,jy3);
 262             dz13             = _mm_sub_pd(iz1,jz3);
 263             dx21             = _mm_sub_pd(ix2,jx1);
 264             dy21             = _mm_sub_pd(iy2,jy1);
 265             dz21             = _mm_sub_pd(iz2,jz1);
 266             dx22             = _mm_sub_pd(ix2,jx2);
 267             dy22             = _mm_sub_pd(iy2,jy2);
 268             dz22             = _mm_sub_pd(iz2,jz2);
 269             dx23             = _mm_sub_pd(ix2,jx3);
 270             dy23             = _mm_sub_pd(iy2,jy3);
 271             dz23             = _mm_sub_pd(iz2,jz3);
 272             dx31             = _mm_sub_pd(ix3,jx1);
 273             dy31             = _mm_sub_pd(iy3,jy1);
 274             dz31             = _mm_sub_pd(iz3,jz1);
 275             dx32             = _mm_sub_pd(ix3,jx2);
 276             dy32             = _mm_sub_pd(iy3,jy2);
 277             dz32             = _mm_sub_pd(iz3,jz2);
 278             dx33             = _mm_sub_pd(ix3,jx3);
 279             dy33             = _mm_sub_pd(iy3,jy3);
 280             dz33             = _mm_sub_pd(iz3,jz3);
 281
 282             /* Calculate squared distance and things based on it */
 283             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 284             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
 285             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
 286             rsq13            = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
 287             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
 288             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
 289             rsq23            = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
 290             rsq31            = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
 291             rsq32            = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
 292             rsq33            = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
 293
 294             rinv00           = avx128fma_invsqrt_d(rsq00);
 295             rinv11           = avx128fma_invsqrt_d(rsq11);
 296             rinv12           = avx128fma_invsqrt_d(rsq12);
 297             rinv13           = avx128fma_invsqrt_d(rsq13);
 298             rinv21           = avx128fma_invsqrt_d(rsq21);
 299             rinv22           = avx128fma_invsqrt_d(rsq22);
 300             rinv23           = avx128fma_invsqrt_d(rsq23);
 301             rinv31           = avx128fma_invsqrt_d(rsq31);
 302             rinv32           = avx128fma_invsqrt_d(rsq32);
 303             rinv33           = avx128fma_invsqrt_d(rsq33);
 304
 305             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
 306             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
 307             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
 308             rinvsq13         = _mm_mul_pd(rinv13,rinv13);
 309             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
 310             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
 311             rinvsq23         = _mm_mul_pd(rinv23,rinv23);
 312             rinvsq31         = _mm_mul_pd(rinv31,rinv31);
 313             rinvsq32         = _mm_mul_pd(rinv32,rinv32);
 314             rinvsq33         = _mm_mul_pd(rinv33,rinv33);
 315
 316             fjx0             = _mm_setzero_pd();
 317             fjy0             = _mm_setzero_pd();
 318             fjz0             = _mm_setzero_pd();
 319             fjx1             = _mm_setzero_pd();
 320             fjy1             = _mm_setzero_pd();
 321             fjz1             = _mm_setzero_pd();
 322             fjx2             = _mm_setzero_pd();
 323             fjy2             = _mm_setzero_pd();
 324             fjz2             = _mm_setzero_pd();
 325             fjx3             = _mm_setzero_pd();
 326             fjy3             = _mm_setzero_pd();
 327             fjz3             = _mm_setzero_pd();
 328
 329             /**************************
 330              * CALCULATE INTERACTIONS *
 331              **************************/
 332
 333             if (gmx_mm_any_lt(rsq00,rcutoff2))
 334             {
 335
 336             r00              = _mm_mul_pd(rsq00,rinv00);
 337
 338             /* Analytical LJ-PME */
 339             rinvsix          = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
 340             ewcljrsq         = _mm_mul_pd(ewclj2,rsq00);
 341             ewclj6           = _mm_mul_pd(ewclj2,_mm_mul_pd(ewclj2,ewclj2));
 342             exponent         = avx128fma_exp_d(ewcljrsq);
 343             /* poly = exp(-(beta*r)^2) * (1 + (beta*r)^2 + (beta*r)^4 /2) */
 344             poly             = _mm_mul_pd(exponent,_mm_macc_pd(_mm_mul_pd(ewcljrsq,ewcljrsq),one_half,_mm_sub_pd(one,ewcljrsq)));
 345             /* vvdw6 = [C6 - C6grid * (1-poly)]/r6 */
 346             vvdw6            = _mm_mul_pd(_mm_macc_pd(-c6grid_00,_mm_sub_pd(one,poly),c6_00),rinvsix);
 347             vvdw12           = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
 348             vvdw             = _mm_msub_pd(_mm_nmacc_pd(c12_00,_mm_mul_pd(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
 349                                _mm_mul_pd(_mm_sub_pd(vvdw6,_mm_macc_pd(c6grid_00,sh_lj_ewald,_mm_mul_pd(c6_00,sh_vdw_invrcut6))),one_sixth));
 350             /* fvdw = vvdw12/r - (vvdw6/r + (C6grid * exponent * beta^6)/r) */
 351             fvdw             = _mm_mul_pd(_mm_add_pd(vvdw12,_mm_msub_pd(_mm_mul_pd(c6grid_00,one_sixth),_mm_mul_pd(exponent,ewclj6),vvdw6)),rinvsq00);
 352
 353             cutoff_mask      = _mm_cmplt_pd(rsq00,rcutoff2);
 354
 355             /* Update potential sum for this i atom from the interaction with this j atom. */
 356             vvdw             = _mm_and_pd(vvdw,cutoff_mask);
 357             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 358
 359             fscal            = fvdw;
 360
 361             fscal            = _mm_and_pd(fscal,cutoff_mask);
 362
 363             /* Update vectorial force */
 364             fix0             = _mm_macc_pd(dx00,fscal,fix0);
 365             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
 366             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
 367
 368             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
 369             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
 370             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
 371
 372             }
 373
 374             /**************************
 375              * CALCULATE INTERACTIONS *
 376              **************************/
 377
 378             if (gmx_mm_any_lt(rsq11,rcutoff2))
 379             {
 380
 381             r11              = _mm_mul_pd(rsq11,rinv11);
 382
 383             /* EWALD ELECTROSTATICS */
 384
 385             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 386             ewrt             = _mm_mul_pd(r11,ewtabscale);
 387             ewitab           = _mm_cvttpd_epi32(ewrt);
 388 #ifdef __XOP__
 389             eweps            = _mm_frcz_pd(ewrt);
 390 #else
 391             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 392 #endif
 393             twoeweps         = _mm_add_pd(eweps,eweps);
 394             ewitab           = _mm_slli_epi32(ewitab,2);
 395             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 396             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 397             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 398             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 399             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 400             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 401             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 402             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 403             velec            = _mm_mul_pd(qq11,_mm_sub_pd(_mm_sub_pd(rinv11,sh_ewald),velec));
 404             felec            = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
 405
 406             cutoff_mask      = _mm_cmplt_pd(rsq11,rcutoff2);
 407
 408             /* Update potential sum for this i atom from the interaction with this j atom. */
 409             velec            = _mm_and_pd(velec,cutoff_mask);
 410             velecsum         = _mm_add_pd(velecsum,velec);
 411
 412             fscal            = felec;
 413
 414             fscal            = _mm_and_pd(fscal,cutoff_mask);
 415
 416             /* Update vectorial force */
 417             fix1             = _mm_macc_pd(dx11,fscal,fix1);
 418             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
 419             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
 420
 421             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
 422             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
 423             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
 424
 425             }
 426
 427             /**************************
 428              * CALCULATE INTERACTIONS *
 429              **************************/
 430
 431             if (gmx_mm_any_lt(rsq12,rcutoff2))
 432             {
 433
 434             r12              = _mm_mul_pd(rsq12,rinv12);
 435
 436             /* EWALD ELECTROSTATICS */
 437
 438             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 439             ewrt             = _mm_mul_pd(r12,ewtabscale);
 440             ewitab           = _mm_cvttpd_epi32(ewrt);
 441 #ifdef __XOP__
 442             eweps            = _mm_frcz_pd(ewrt);
 443 #else
 444             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 445 #endif
 446             twoeweps         = _mm_add_pd(eweps,eweps);
 447             ewitab           = _mm_slli_epi32(ewitab,2);
 448             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 449             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 450             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 451             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 452             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 453             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 454             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 455             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 456             velec            = _mm_mul_pd(qq12,_mm_sub_pd(_mm_sub_pd(rinv12,sh_ewald),velec));
 457             felec            = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
 458
 459             cutoff_mask      = _mm_cmplt_pd(rsq12,rcutoff2);
 460
 461             /* Update potential sum for this i atom from the interaction with this j atom. */
 462             velec            = _mm_and_pd(velec,cutoff_mask);
 463             velecsum         = _mm_add_pd(velecsum,velec);
 464
 465             fscal            = felec;
 466
 467             fscal            = _mm_and_pd(fscal,cutoff_mask);
 468
 469             /* Update vectorial force */
 470             fix1             = _mm_macc_pd(dx12,fscal,fix1);
 471             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
 472             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
 473
 474             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
 475             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
 476             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
 477
 478             }
 479
 480             /**************************
 481              * CALCULATE INTERACTIONS *
 482              **************************/
 483
 484             if (gmx_mm_any_lt(rsq13,rcutoff2))
 485             {
 486
 487             r13              = _mm_mul_pd(rsq13,rinv13);
 488
 489             /* EWALD ELECTROSTATICS */
 490
 491             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 492             ewrt             = _mm_mul_pd(r13,ewtabscale);
 493             ewitab           = _mm_cvttpd_epi32(ewrt);
 494 #ifdef __XOP__
 495             eweps            = _mm_frcz_pd(ewrt);
 496 #else
 497             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 498 #endif
 499             twoeweps         = _mm_add_pd(eweps,eweps);
 500             ewitab           = _mm_slli_epi32(ewitab,2);
 501             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 502             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 503             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 504             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 505             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 506             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 507             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 508             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 509             velec            = _mm_mul_pd(qq13,_mm_sub_pd(_mm_sub_pd(rinv13,sh_ewald),velec));
 510             felec            = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
 511
 512             cutoff_mask      = _mm_cmplt_pd(rsq13,rcutoff2);
 513
 514             /* Update potential sum for this i atom from the interaction with this j atom. */
 515             velec            = _mm_and_pd(velec,cutoff_mask);
 516             velecsum         = _mm_add_pd(velecsum,velec);
 517
 518             fscal            = felec;
 519
 520             fscal            = _mm_and_pd(fscal,cutoff_mask);
 521
 522             /* Update vectorial force */
 523             fix1             = _mm_macc_pd(dx13,fscal,fix1);
 524             fiy1             = _mm_macc_pd(dy13,fscal,fiy1);
 525             fiz1             = _mm_macc_pd(dz13,fscal,fiz1);
 526
 527             fjx3             = _mm_macc_pd(dx13,fscal,fjx3);
 528             fjy3             = _mm_macc_pd(dy13,fscal,fjy3);
 529             fjz3             = _mm_macc_pd(dz13,fscal,fjz3);
 530
 531             }
 532
 533             /**************************
 534              * CALCULATE INTERACTIONS *
 535              **************************/
 536
 537             if (gmx_mm_any_lt(rsq21,rcutoff2))
 538             {
 539
 540             r21              = _mm_mul_pd(rsq21,rinv21);
 541
 542             /* EWALD ELECTROSTATICS */
 543
 544             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 545             ewrt             = _mm_mul_pd(r21,ewtabscale);
 546             ewitab           = _mm_cvttpd_epi32(ewrt);
 547 #ifdef __XOP__
 548             eweps            = _mm_frcz_pd(ewrt);
 549 #else
 550             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 551 #endif
 552             twoeweps         = _mm_add_pd(eweps,eweps);
 553             ewitab           = _mm_slli_epi32(ewitab,2);
 554             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 555             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 556             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 557             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 558             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 559             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 560             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 561             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 562             velec            = _mm_mul_pd(qq21,_mm_sub_pd(_mm_sub_pd(rinv21,sh_ewald),velec));
 563             felec            = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
 564
 565             cutoff_mask      = _mm_cmplt_pd(rsq21,rcutoff2);
 566
 567             /* Update potential sum for this i atom from the interaction with this j atom. */
 568             velec            = _mm_and_pd(velec,cutoff_mask);
 569             velecsum         = _mm_add_pd(velecsum,velec);
 570
 571             fscal            = felec;
 572
 573             fscal            = _mm_and_pd(fscal,cutoff_mask);
 574
 575             /* Update vectorial force */
 576             fix2             = _mm_macc_pd(dx21,fscal,fix2);
 577             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
 578             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
 579
 580             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
 581             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
 582             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
 583
 584             }
 585
 586             /**************************
 587              * CALCULATE INTERACTIONS *
 588              **************************/
 589
 590             if (gmx_mm_any_lt(rsq22,rcutoff2))
 591             {
 592
 593             r22              = _mm_mul_pd(rsq22,rinv22);
 594
 595             /* EWALD ELECTROSTATICS */
 596
 597             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 598             ewrt             = _mm_mul_pd(r22,ewtabscale);
 599             ewitab           = _mm_cvttpd_epi32(ewrt);
 600 #ifdef __XOP__
 601             eweps            = _mm_frcz_pd(ewrt);
 602 #else
 603             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 604 #endif
 605             twoeweps         = _mm_add_pd(eweps,eweps);
 606             ewitab           = _mm_slli_epi32(ewitab,2);
 607             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 608             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 609             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 610             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 611             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 612             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 613             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 614             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 615             velec            = _mm_mul_pd(qq22,_mm_sub_pd(_mm_sub_pd(rinv22,sh_ewald),velec));
 616             felec            = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
 617
 618             cutoff_mask      = _mm_cmplt_pd(rsq22,rcutoff2);
 619
 620             /* Update potential sum for this i atom from the interaction with this j atom. */
 621             velec            = _mm_and_pd(velec,cutoff_mask);
 622             velecsum         = _mm_add_pd(velecsum,velec);
 623
 624             fscal            = felec;
 625
 626             fscal            = _mm_and_pd(fscal,cutoff_mask);
 627
 628             /* Update vectorial force */
 629             fix2             = _mm_macc_pd(dx22,fscal,fix2);
 630             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
 631             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
 632
 633             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
 634             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
 635             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
 636
 637             }
 638
 639             /**************************
 640              * CALCULATE INTERACTIONS *
 641              **************************/
 642
 643             if (gmx_mm_any_lt(rsq23,rcutoff2))
 644             {
 645
 646             r23              = _mm_mul_pd(rsq23,rinv23);
 647
 648             /* EWALD ELECTROSTATICS */
 649
 650             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 651             ewrt             = _mm_mul_pd(r23,ewtabscale);
 652             ewitab           = _mm_cvttpd_epi32(ewrt);
 653 #ifdef __XOP__
 654             eweps            = _mm_frcz_pd(ewrt);
 655 #else
 656             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 657 #endif
 658             twoeweps         = _mm_add_pd(eweps,eweps);
 659             ewitab           = _mm_slli_epi32(ewitab,2);
 660             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 661             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 662             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 663             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 664             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 665             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 666             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 667             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 668             velec            = _mm_mul_pd(qq23,_mm_sub_pd(_mm_sub_pd(rinv23,sh_ewald),velec));
 669             felec            = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
 670
 671             cutoff_mask      = _mm_cmplt_pd(rsq23,rcutoff2);
 672
 673             /* Update potential sum for this i atom from the interaction with this j atom. */
 674             velec            = _mm_and_pd(velec,cutoff_mask);
 675             velecsum         = _mm_add_pd(velecsum,velec);
 676
 677             fscal            = felec;
 678
 679             fscal            = _mm_and_pd(fscal,cutoff_mask);
 680
 681             /* Update vectorial force */
 682             fix2             = _mm_macc_pd(dx23,fscal,fix2);
 683             fiy2             = _mm_macc_pd(dy23,fscal,fiy2);
 684             fiz2             = _mm_macc_pd(dz23,fscal,fiz2);
 685
 686             fjx3             = _mm_macc_pd(dx23,fscal,fjx3);
 687             fjy3             = _mm_macc_pd(dy23,fscal,fjy3);
 688             fjz3             = _mm_macc_pd(dz23,fscal,fjz3);
 689
 690             }
 691
 692             /**************************
 693              * CALCULATE INTERACTIONS *
 694              **************************/
 695
 696             if (gmx_mm_any_lt(rsq31,rcutoff2))
 697             {
 698
 699             r31              = _mm_mul_pd(rsq31,rinv31);
 700
 701             /* EWALD ELECTROSTATICS */
 702
 703             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 704             ewrt             = _mm_mul_pd(r31,ewtabscale);
 705             ewitab           = _mm_cvttpd_epi32(ewrt);
 706 #ifdef __XOP__
 707             eweps            = _mm_frcz_pd(ewrt);
 708 #else
 709             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 710 #endif
 711             twoeweps         = _mm_add_pd(eweps,eweps);
 712             ewitab           = _mm_slli_epi32(ewitab,2);
 713             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 714             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 715             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 716             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 717             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 718             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 719             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 720             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 721             velec            = _mm_mul_pd(qq31,_mm_sub_pd(_mm_sub_pd(rinv31,sh_ewald),velec));
 722             felec            = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
 723
 724             cutoff_mask      = _mm_cmplt_pd(rsq31,rcutoff2);
 725
 726             /* Update potential sum for this i atom from the interaction with this j atom. */
 727             velec            = _mm_and_pd(velec,cutoff_mask);
 728             velecsum         = _mm_add_pd(velecsum,velec);
 729
 730             fscal            = felec;
 731
 732             fscal            = _mm_and_pd(fscal,cutoff_mask);
 733
 734             /* Update vectorial force */
 735             fix3             = _mm_macc_pd(dx31,fscal,fix3);
 736             fiy3             = _mm_macc_pd(dy31,fscal,fiy3);
 737             fiz3             = _mm_macc_pd(dz31,fscal,fiz3);
 738
 739             fjx1             = _mm_macc_pd(dx31,fscal,fjx1);
 740             fjy1             = _mm_macc_pd(dy31,fscal,fjy1);
 741             fjz1             = _mm_macc_pd(dz31,fscal,fjz1);
 742
 743             }
 744
 745             /**************************
 746              * CALCULATE INTERACTIONS *
 747              **************************/
 748
 749             if (gmx_mm_any_lt(rsq32,rcutoff2))
 750             {
 751
 752             r32              = _mm_mul_pd(rsq32,rinv32);
 753
 754             /* EWALD ELECTROSTATICS */
 755
 756             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 757             ewrt             = _mm_mul_pd(r32,ewtabscale);
 758             ewitab           = _mm_cvttpd_epi32(ewrt);
 759 #ifdef __XOP__
 760             eweps            = _mm_frcz_pd(ewrt);
 761 #else
 762             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 763 #endif
 764             twoeweps         = _mm_add_pd(eweps,eweps);
 765             ewitab           = _mm_slli_epi32(ewitab,2);
 766             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 767             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 768             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 769             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 770             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 771             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 772             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 773             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 774             velec            = _mm_mul_pd(qq32,_mm_sub_pd(_mm_sub_pd(rinv32,sh_ewald),velec));
 775             felec            = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
 776
 777             cutoff_mask      = _mm_cmplt_pd(rsq32,rcutoff2);
 778
 779             /* Update potential sum for this i atom from the interaction with this j atom. */
 780             velec            = _mm_and_pd(velec,cutoff_mask);
 781             velecsum         = _mm_add_pd(velecsum,velec);
 782
 783             fscal            = felec;
 784
 785             fscal            = _mm_and_pd(fscal,cutoff_mask);
 786
 787             /* Update vectorial force */
 788             fix3             = _mm_macc_pd(dx32,fscal,fix3);
 789             fiy3             = _mm_macc_pd(dy32,fscal,fiy3);
 790             fiz3             = _mm_macc_pd(dz32,fscal,fiz3);
 791
 792             fjx2             = _mm_macc_pd(dx32,fscal,fjx2);
 793             fjy2             = _mm_macc_pd(dy32,fscal,fjy2);
 794             fjz2             = _mm_macc_pd(dz32,fscal,fjz2);
 795
 796             }
 797
 798             /**************************
 799              * CALCULATE INTERACTIONS *
 800              **************************/
 801
 802             if (gmx_mm_any_lt(rsq33,rcutoff2))
 803             {
 804
 805             r33              = _mm_mul_pd(rsq33,rinv33);
 806
 807             /* EWALD ELECTROSTATICS */
 808
 809             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
 810             ewrt             = _mm_mul_pd(r33,ewtabscale);
 811             ewitab           = _mm_cvttpd_epi32(ewrt);
 812 #ifdef __XOP__
 813             eweps            = _mm_frcz_pd(ewrt);
 814 #else
 815             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
 816 #endif
 817             twoeweps         = _mm_add_pd(eweps,eweps);
 818             ewitab           = _mm_slli_epi32(ewitab,2);
 819             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
 820             ewtabD           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
 821             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
 822             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
 823             ewtabFn          = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
 824             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
 825             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
 826             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
 827             velec            = _mm_mul_pd(qq33,_mm_sub_pd(_mm_sub_pd(rinv33,sh_ewald),velec));
 828             felec            = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
 829
 830             cutoff_mask      = _mm_cmplt_pd(rsq33,rcutoff2);
 831
 832             /* Update potential sum for this i atom from the interaction with this j atom. */
 833             velec            = _mm_and_pd(velec,cutoff_mask);
 834             velecsum         = _mm_add_pd(velecsum,velec);
 835
 836             fscal            = felec;
 837
 838             fscal            = _mm_and_pd(fscal,cutoff_mask);
 839
 840             /* Update vectorial force */
 841             fix3             = _mm_macc_pd(dx33,fscal,fix3);
 842             fiy3             = _mm_macc_pd(dy33,fscal,fiy3);
 843             fiz3             = _mm_macc_pd(dz33,fscal,fiz3);
 844
 845             fjx3             = _mm_macc_pd(dx33,fscal,fjx3);
 846             fjy3             = _mm_macc_pd(dy33,fscal,fjy3);
 847             fjz3             = _mm_macc_pd(dz33,fscal,fjz3);
 848
 849             }
 850
 851             gmx_mm_decrement_4rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
 852
 853             /* Inner loop uses 502 flops */
 854         }
 855
 856         if(jidx<j_index_end)
 857         {
 858
 859             jnrA             = jjnr[jidx];
 860             j_coord_offsetA  = DIM*jnrA;
 861
 862             /* load j atom coordinates */
 863             gmx_mm_load_4rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
 864                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
 865                                               &jy2,&jz2,&jx3,&jy3,&jz3);
 866
 867             /* Calculate displacement vector */
 868             dx00             = _mm_sub_pd(ix0,jx0);
 869             dy00             = _mm_sub_pd(iy0,jy0);
 870             dz00             = _mm_sub_pd(iz0,jz0);
 871             dx11             = _mm_sub_pd(ix1,jx1);
 872             dy11             = _mm_sub_pd(iy1,jy1);
 873             dz11             = _mm_sub_pd(iz1,jz1);
 874             dx12             = _mm_sub_pd(ix1,jx2);
 875             dy12             = _mm_sub_pd(iy1,jy2);
 876             dz12             = _mm_sub_pd(iz1,jz2);
 877             dx13             = _mm_sub_pd(ix1,jx3);
 878             dy13             = _mm_sub_pd(iy1,jy3);
 879             dz13             = _mm_sub_pd(iz1,jz3);
 880             dx21             = _mm_sub_pd(ix2,jx1);
 881             dy21             = _mm_sub_pd(iy2,jy1);
 882             dz21             = _mm_sub_pd(iz2,jz1);
 883             dx22             = _mm_sub_pd(ix2,jx2);
 884             dy22             = _mm_sub_pd(iy2,jy2);
 885             dz22             = _mm_sub_pd(iz2,jz2);
 886             dx23             = _mm_sub_pd(ix2,jx3);
 887             dy23             = _mm_sub_pd(iy2,jy3);
 888             dz23             = _mm_sub_pd(iz2,jz3);
 889             dx31             = _mm_sub_pd(ix3,jx1);
 890             dy31             = _mm_sub_pd(iy3,jy1);
 891             dz31             = _mm_sub_pd(iz3,jz1);
 892             dx32             = _mm_sub_pd(ix3,jx2);
 893             dy32             = _mm_sub_pd(iy3,jy2);
 894             dz32             = _mm_sub_pd(iz3,jz2);
 895             dx33             = _mm_sub_pd(ix3,jx3);
 896             dy33             = _mm_sub_pd(iy3,jy3);
 897             dz33             = _mm_sub_pd(iz3,jz3);
 898
 899             /* Calculate squared distance and things based on it */
 900             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 901             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
 902             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
 903             rsq13            = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
 904             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
 905             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
 906             rsq23            = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
 907             rsq31            = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
 908             rsq32            = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
 909             rsq33            = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
 910
 911             rinv00           = avx128fma_invsqrt_d(rsq00);
 912             rinv11           = avx128fma_invsqrt_d(rsq11);
 913             rinv12           = avx128fma_invsqrt_d(rsq12);
 914             rinv13           = avx128fma_invsqrt_d(rsq13);
 915             rinv21           = avx128fma_invsqrt_d(rsq21);
 916             rinv22           = avx128fma_invsqrt_d(rsq22);
 917             rinv23           = avx128fma_invsqrt_d(rsq23);
 918             rinv31           = avx128fma_invsqrt_d(rsq31);
 919             rinv32           = avx128fma_invsqrt_d(rsq32);
 920             rinv33           = avx128fma_invsqrt_d(rsq33);
 921
 922             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
 923             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
 924             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
 925             rinvsq13         = _mm_mul_pd(rinv13,rinv13);
 926             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
 927             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
 928             rinvsq23         = _mm_mul_pd(rinv23,rinv23);
 929             rinvsq31         = _mm_mul_pd(rinv31,rinv31);
 930             rinvsq32         = _mm_mul_pd(rinv32,rinv32);
 931             rinvsq33         = _mm_mul_pd(rinv33,rinv33);
 932
 933             fjx0             = _mm_setzero_pd();
 934             fjy0             = _mm_setzero_pd();
 935             fjz0             = _mm_setzero_pd();
 936             fjx1             = _mm_setzero_pd();
 937             fjy1             = _mm_setzero_pd();
 938             fjz1             = _mm_setzero_pd();
 939             fjx2             = _mm_setzero_pd();
 940             fjy2             = _mm_setzero_pd();
 941             fjz2             = _mm_setzero_pd();
 942             fjx3             = _mm_setzero_pd();
 943             fjy3             = _mm_setzero_pd();
 944             fjz3             = _mm_setzero_pd();
 945
 946             /**************************
 947              * CALCULATE INTERACTIONS *
 948              **************************/
 949
 950             if (gmx_mm_any_lt(rsq00,rcutoff2))
 951             {
 952
 953             r00              = _mm_mul_pd(rsq00,rinv00);
 954
 955             /* Analytical LJ-PME */
 956             rinvsix          = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
 957             ewcljrsq         = _mm_mul_pd(ewclj2,rsq00);
 958             ewclj6           = _mm_mul_pd(ewclj2,_mm_mul_pd(ewclj2,ewclj2));
 959             exponent         = avx128fma_exp_d(ewcljrsq);
 960             /* poly = exp(-(beta*r)^2) * (1 + (beta*r)^2 + (beta*r)^4 /2) */
 961             poly             = _mm_mul_pd(exponent,_mm_macc_pd(_mm_mul_pd(ewcljrsq,ewcljrsq),one_half,_mm_sub_pd(one,ewcljrsq)));
 962             /* vvdw6 = [C6 - C6grid * (1-poly)]/r6 */
 963             vvdw6            = _mm_mul_pd(_mm_macc_pd(-c6grid_00,_mm_sub_pd(one,poly),c6_00),rinvsix);
 964             vvdw12           = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
 965             vvdw             = _mm_msub_pd(_mm_nmacc_pd(c12_00,_mm_mul_pd(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
 966                                _mm_mul_pd(_mm_sub_pd(vvdw6,_mm_macc_pd(c6grid_00,sh_lj_ewald,_mm_mul_pd(c6_00,sh_vdw_invrcut6))),one_sixth));
 967             /* fvdw = vvdw12/r - (vvdw6/r + (C6grid * exponent * beta^6)/r) */
 968             fvdw             = _mm_mul_pd(_mm_add_pd(vvdw12,_mm_msub_pd(_mm_mul_pd(c6grid_00,one_sixth),_mm_mul_pd(exponent,ewclj6),vvdw6)),rinvsq00);
 969
 970             cutoff_mask      = _mm_cmplt_pd(rsq00,rcutoff2);
 971
 972             /* Update potential sum for this i atom from the interaction with this j atom. */
 973             vvdw             = _mm_and_pd(vvdw,cutoff_mask);
 974             vvdw             = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
 975             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 976
 977             fscal            = fvdw;
 978
 979             fscal            = _mm_and_pd(fscal,cutoff_mask);
 980
 981             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 982
 983             /* Update vectorial force */
 984             fix0             = _mm_macc_pd(dx00,fscal,fix0);
 985             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
 986             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
 987
 988             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
 989             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
 990             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
 991
 992             }
 993
 994             /**************************
 995              * CALCULATE INTERACTIONS *
 996              **************************/
 997
 998             if (gmx_mm_any_lt(rsq11,rcutoff2))
 999             {
1000
1001             r11              = _mm_mul_pd(rsq11,rinv11);
1002
1003             /* EWALD ELECTROSTATICS */
1004
1005             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1006             ewrt             = _mm_mul_pd(r11,ewtabscale);
1007             ewitab           = _mm_cvttpd_epi32(ewrt);
1008 #ifdef __XOP__
1009             eweps            = _mm_frcz_pd(ewrt);
1010 #else
1011             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1012 #endif
1013             twoeweps         = _mm_add_pd(eweps,eweps);
1014             ewitab           = _mm_slli_epi32(ewitab,2);
1015             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1016             ewtabD           = _mm_setzero_pd();
1017             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1018             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1019             ewtabFn          = _mm_setzero_pd();
1020             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1021             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1022             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1023             velec            = _mm_mul_pd(qq11,_mm_sub_pd(_mm_sub_pd(rinv11,sh_ewald),velec));
1024             felec            = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
1025
1026             cutoff_mask      = _mm_cmplt_pd(rsq11,rcutoff2);
1027
1028             /* Update potential sum for this i atom from the interaction with this j atom. */
1029             velec            = _mm_and_pd(velec,cutoff_mask);
1030             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1031             velecsum         = _mm_add_pd(velecsum,velec);
1032
1033             fscal            = felec;
1034
1035             fscal            = _mm_and_pd(fscal,cutoff_mask);
1036
1037             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1038
1039             /* Update vectorial force */
1040             fix1             = _mm_macc_pd(dx11,fscal,fix1);
1041             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
1042             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
1043
1044             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
1045             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
1046             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
1047
1048             }
1049
1050             /**************************
1051              * CALCULATE INTERACTIONS *
1052              **************************/
1053
1054             if (gmx_mm_any_lt(rsq12,rcutoff2))
1055             {
1056
1057             r12              = _mm_mul_pd(rsq12,rinv12);
1058
1059             /* EWALD ELECTROSTATICS */
1060
1061             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1062             ewrt             = _mm_mul_pd(r12,ewtabscale);
1063             ewitab           = _mm_cvttpd_epi32(ewrt);
1064 #ifdef __XOP__
1065             eweps            = _mm_frcz_pd(ewrt);
1066 #else
1067             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1068 #endif
1069             twoeweps         = _mm_add_pd(eweps,eweps);
1070             ewitab           = _mm_slli_epi32(ewitab,2);
1071             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1072             ewtabD           = _mm_setzero_pd();
1073             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1074             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1075             ewtabFn          = _mm_setzero_pd();
1076             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1077             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1078             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1079             velec            = _mm_mul_pd(qq12,_mm_sub_pd(_mm_sub_pd(rinv12,sh_ewald),velec));
1080             felec            = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
1081
1082             cutoff_mask      = _mm_cmplt_pd(rsq12,rcutoff2);
1083
1084             /* Update potential sum for this i atom from the interaction with this j atom. */
1085             velec            = _mm_and_pd(velec,cutoff_mask);
1086             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1087             velecsum         = _mm_add_pd(velecsum,velec);
1088
1089             fscal            = felec;
1090
1091             fscal            = _mm_and_pd(fscal,cutoff_mask);
1092
1093             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1094
1095             /* Update vectorial force */
1096             fix1             = _mm_macc_pd(dx12,fscal,fix1);
1097             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
1098             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
1099
1100             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
1101             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
1102             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
1103
1104             }
1105
1106             /**************************
1107              * CALCULATE INTERACTIONS *
1108              **************************/
1109
1110             if (gmx_mm_any_lt(rsq13,rcutoff2))
1111             {
1112
1113             r13              = _mm_mul_pd(rsq13,rinv13);
1114
1115             /* EWALD ELECTROSTATICS */
1116
1117             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1118             ewrt             = _mm_mul_pd(r13,ewtabscale);
1119             ewitab           = _mm_cvttpd_epi32(ewrt);
1120 #ifdef __XOP__
1121             eweps            = _mm_frcz_pd(ewrt);
1122 #else
1123             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1124 #endif
1125             twoeweps         = _mm_add_pd(eweps,eweps);
1126             ewitab           = _mm_slli_epi32(ewitab,2);
1127             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1128             ewtabD           = _mm_setzero_pd();
1129             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1130             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1131             ewtabFn          = _mm_setzero_pd();
1132             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1133             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1134             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1135             velec            = _mm_mul_pd(qq13,_mm_sub_pd(_mm_sub_pd(rinv13,sh_ewald),velec));
1136             felec            = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
1137
1138             cutoff_mask      = _mm_cmplt_pd(rsq13,rcutoff2);
1139
1140             /* Update potential sum for this i atom from the interaction with this j atom. */
1141             velec            = _mm_and_pd(velec,cutoff_mask);
1142             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1143             velecsum         = _mm_add_pd(velecsum,velec);
1144
1145             fscal            = felec;
1146
1147             fscal            = _mm_and_pd(fscal,cutoff_mask);
1148
1149             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1150
1151             /* Update vectorial force */
1152             fix1             = _mm_macc_pd(dx13,fscal,fix1);
1153             fiy1             = _mm_macc_pd(dy13,fscal,fiy1);
1154             fiz1             = _mm_macc_pd(dz13,fscal,fiz1);
1155
1156             fjx3             = _mm_macc_pd(dx13,fscal,fjx3);
1157             fjy3             = _mm_macc_pd(dy13,fscal,fjy3);
1158             fjz3             = _mm_macc_pd(dz13,fscal,fjz3);
1159
1160             }
1161
1162             /**************************
1163              * CALCULATE INTERACTIONS *
1164              **************************/
1165
1166             if (gmx_mm_any_lt(rsq21,rcutoff2))
1167             {
1168
1169             r21              = _mm_mul_pd(rsq21,rinv21);
1170
1171             /* EWALD ELECTROSTATICS */
1172
1173             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1174             ewrt             = _mm_mul_pd(r21,ewtabscale);
1175             ewitab           = _mm_cvttpd_epi32(ewrt);
1176 #ifdef __XOP__
1177             eweps            = _mm_frcz_pd(ewrt);
1178 #else
1179             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1180 #endif
1181             twoeweps         = _mm_add_pd(eweps,eweps);
1182             ewitab           = _mm_slli_epi32(ewitab,2);
1183             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1184             ewtabD           = _mm_setzero_pd();
1185             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1186             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1187             ewtabFn          = _mm_setzero_pd();
1188             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1189             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1190             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1191             velec            = _mm_mul_pd(qq21,_mm_sub_pd(_mm_sub_pd(rinv21,sh_ewald),velec));
1192             felec            = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
1193
1194             cutoff_mask      = _mm_cmplt_pd(rsq21,rcutoff2);
1195
1196             /* Update potential sum for this i atom from the interaction with this j atom. */
1197             velec            = _mm_and_pd(velec,cutoff_mask);
1198             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1199             velecsum         = _mm_add_pd(velecsum,velec);
1200
1201             fscal            = felec;
1202
1203             fscal            = _mm_and_pd(fscal,cutoff_mask);
1204
1205             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1206
1207             /* Update vectorial force */
1208             fix2             = _mm_macc_pd(dx21,fscal,fix2);
1209             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
1210             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
1211
1212             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
1213             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
1214             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
1215
1216             }
1217
1218             /**************************
1219              * CALCULATE INTERACTIONS *
1220              **************************/
1221
1222             if (gmx_mm_any_lt(rsq22,rcutoff2))
1223             {
1224
1225             r22              = _mm_mul_pd(rsq22,rinv22);
1226
1227             /* EWALD ELECTROSTATICS */
1228
1229             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1230             ewrt             = _mm_mul_pd(r22,ewtabscale);
1231             ewitab           = _mm_cvttpd_epi32(ewrt);
1232 #ifdef __XOP__
1233             eweps            = _mm_frcz_pd(ewrt);
1234 #else
1235             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1236 #endif
1237             twoeweps         = _mm_add_pd(eweps,eweps);
1238             ewitab           = _mm_slli_epi32(ewitab,2);
1239             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1240             ewtabD           = _mm_setzero_pd();
1241             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1242             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1243             ewtabFn          = _mm_setzero_pd();
1244             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1245             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1246             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1247             velec            = _mm_mul_pd(qq22,_mm_sub_pd(_mm_sub_pd(rinv22,sh_ewald),velec));
1248             felec            = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
1249
1250             cutoff_mask      = _mm_cmplt_pd(rsq22,rcutoff2);
1251
1252             /* Update potential sum for this i atom from the interaction with this j atom. */
1253             velec            = _mm_and_pd(velec,cutoff_mask);
1254             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1255             velecsum         = _mm_add_pd(velecsum,velec);
1256
1257             fscal            = felec;
1258
1259             fscal            = _mm_and_pd(fscal,cutoff_mask);
1260
1261             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1262
1263             /* Update vectorial force */
1264             fix2             = _mm_macc_pd(dx22,fscal,fix2);
1265             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
1266             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
1267
1268             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
1269             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
1270             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
1271
1272             }
1273
1274             /**************************
1275              * CALCULATE INTERACTIONS *
1276              **************************/
1277
1278             if (gmx_mm_any_lt(rsq23,rcutoff2))
1279             {
1280
1281             r23              = _mm_mul_pd(rsq23,rinv23);
1282
1283             /* EWALD ELECTROSTATICS */
1284
1285             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1286             ewrt             = _mm_mul_pd(r23,ewtabscale);
1287             ewitab           = _mm_cvttpd_epi32(ewrt);
1288 #ifdef __XOP__
1289             eweps            = _mm_frcz_pd(ewrt);
1290 #else
1291             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1292 #endif
1293             twoeweps         = _mm_add_pd(eweps,eweps);
1294             ewitab           = _mm_slli_epi32(ewitab,2);
1295             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1296             ewtabD           = _mm_setzero_pd();
1297             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1298             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1299             ewtabFn          = _mm_setzero_pd();
1300             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1301             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1302             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1303             velec            = _mm_mul_pd(qq23,_mm_sub_pd(_mm_sub_pd(rinv23,sh_ewald),velec));
1304             felec            = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
1305
1306             cutoff_mask      = _mm_cmplt_pd(rsq23,rcutoff2);
1307
1308             /* Update potential sum for this i atom from the interaction with this j atom. */
1309             velec            = _mm_and_pd(velec,cutoff_mask);
1310             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1311             velecsum         = _mm_add_pd(velecsum,velec);
1312
1313             fscal            = felec;
1314
1315             fscal            = _mm_and_pd(fscal,cutoff_mask);
1316
1317             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1318
1319             /* Update vectorial force */
1320             fix2             = _mm_macc_pd(dx23,fscal,fix2);
1321             fiy2             = _mm_macc_pd(dy23,fscal,fiy2);
1322             fiz2             = _mm_macc_pd(dz23,fscal,fiz2);
1323
1324             fjx3             = _mm_macc_pd(dx23,fscal,fjx3);
1325             fjy3             = _mm_macc_pd(dy23,fscal,fjy3);
1326             fjz3             = _mm_macc_pd(dz23,fscal,fjz3);
1327
1328             }
1329
1330             /**************************
1331              * CALCULATE INTERACTIONS *
1332              **************************/
1333
1334             if (gmx_mm_any_lt(rsq31,rcutoff2))
1335             {
1336
1337             r31              = _mm_mul_pd(rsq31,rinv31);
1338
1339             /* EWALD ELECTROSTATICS */
1340
1341             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1342             ewrt             = _mm_mul_pd(r31,ewtabscale);
1343             ewitab           = _mm_cvttpd_epi32(ewrt);
1344 #ifdef __XOP__
1345             eweps            = _mm_frcz_pd(ewrt);
1346 #else
1347             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1348 #endif
1349             twoeweps         = _mm_add_pd(eweps,eweps);
1350             ewitab           = _mm_slli_epi32(ewitab,2);
1351             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1352             ewtabD           = _mm_setzero_pd();
1353             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1354             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1355             ewtabFn          = _mm_setzero_pd();
1356             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1357             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1358             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1359             velec            = _mm_mul_pd(qq31,_mm_sub_pd(_mm_sub_pd(rinv31,sh_ewald),velec));
1360             felec            = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
1361
1362             cutoff_mask      = _mm_cmplt_pd(rsq31,rcutoff2);
1363
1364             /* Update potential sum for this i atom from the interaction with this j atom. */
1365             velec            = _mm_and_pd(velec,cutoff_mask);
1366             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1367             velecsum         = _mm_add_pd(velecsum,velec);
1368
1369             fscal            = felec;
1370
1371             fscal            = _mm_and_pd(fscal,cutoff_mask);
1372
1373             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1374
1375             /* Update vectorial force */
1376             fix3             = _mm_macc_pd(dx31,fscal,fix3);
1377             fiy3             = _mm_macc_pd(dy31,fscal,fiy3);
1378             fiz3             = _mm_macc_pd(dz31,fscal,fiz3);
1379
1380             fjx1             = _mm_macc_pd(dx31,fscal,fjx1);
1381             fjy1             = _mm_macc_pd(dy31,fscal,fjy1);
1382             fjz1             = _mm_macc_pd(dz31,fscal,fjz1);
1383
1384             }
1385
1386             /**************************
1387              * CALCULATE INTERACTIONS *
1388              **************************/
1389
1390             if (gmx_mm_any_lt(rsq32,rcutoff2))
1391             {
1392
1393             r32              = _mm_mul_pd(rsq32,rinv32);
1394
1395             /* EWALD ELECTROSTATICS */
1396
1397             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1398             ewrt             = _mm_mul_pd(r32,ewtabscale);
1399             ewitab           = _mm_cvttpd_epi32(ewrt);
1400 #ifdef __XOP__
1401             eweps            = _mm_frcz_pd(ewrt);
1402 #else
1403             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1404 #endif
1405             twoeweps         = _mm_add_pd(eweps,eweps);
1406             ewitab           = _mm_slli_epi32(ewitab,2);
1407             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1408             ewtabD           = _mm_setzero_pd();
1409             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1410             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1411             ewtabFn          = _mm_setzero_pd();
1412             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1413             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1414             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1415             velec            = _mm_mul_pd(qq32,_mm_sub_pd(_mm_sub_pd(rinv32,sh_ewald),velec));
1416             felec            = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
1417
1418             cutoff_mask      = _mm_cmplt_pd(rsq32,rcutoff2);
1419
1420             /* Update potential sum for this i atom from the interaction with this j atom. */
1421             velec            = _mm_and_pd(velec,cutoff_mask);
1422             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1423             velecsum         = _mm_add_pd(velecsum,velec);
1424
1425             fscal            = felec;
1426
1427             fscal            = _mm_and_pd(fscal,cutoff_mask);
1428
1429             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1430
1431             /* Update vectorial force */
1432             fix3             = _mm_macc_pd(dx32,fscal,fix3);
1433             fiy3             = _mm_macc_pd(dy32,fscal,fiy3);
1434             fiz3             = _mm_macc_pd(dz32,fscal,fiz3);
1435
1436             fjx2             = _mm_macc_pd(dx32,fscal,fjx2);
1437             fjy2             = _mm_macc_pd(dy32,fscal,fjy2);
1438             fjz2             = _mm_macc_pd(dz32,fscal,fjz2);
1439
1440             }
1441
1442             /**************************
1443              * CALCULATE INTERACTIONS *
1444              **************************/
1445
1446             if (gmx_mm_any_lt(rsq33,rcutoff2))
1447             {
1448
1449             r33              = _mm_mul_pd(rsq33,rinv33);
1450
1451             /* EWALD ELECTROSTATICS */
1452
1453             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1454             ewrt             = _mm_mul_pd(r33,ewtabscale);
1455             ewitab           = _mm_cvttpd_epi32(ewrt);
1456 #ifdef __XOP__
1457             eweps            = _mm_frcz_pd(ewrt);
1458 #else
1459             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1460 #endif
1461             twoeweps         = _mm_add_pd(eweps,eweps);
1462             ewitab           = _mm_slli_epi32(ewitab,2);
1463             ewtabF           = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1464             ewtabD           = _mm_setzero_pd();
1465             GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1466             ewtabV           = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1467             ewtabFn          = _mm_setzero_pd();
1468             GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1469             felec            = _mm_macc_pd(eweps,ewtabD,ewtabF);
1470             velec            = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1471             velec            = _mm_mul_pd(qq33,_mm_sub_pd(_mm_sub_pd(rinv33,sh_ewald),velec));
1472             felec            = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
1473
1474             cutoff_mask      = _mm_cmplt_pd(rsq33,rcutoff2);
1475
1476             /* Update potential sum for this i atom from the interaction with this j atom. */
1477             velec            = _mm_and_pd(velec,cutoff_mask);
1478             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1479             velecsum         = _mm_add_pd(velecsum,velec);
1480
1481             fscal            = felec;
1482
1483             fscal            = _mm_and_pd(fscal,cutoff_mask);
1484
1485             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1486
1487             /* Update vectorial force */
1488             fix3             = _mm_macc_pd(dx33,fscal,fix3);
1489             fiy3             = _mm_macc_pd(dy33,fscal,fiy3);
1490             fiz3             = _mm_macc_pd(dz33,fscal,fiz3);
1491
1492             fjx3             = _mm_macc_pd(dx33,fscal,fjx3);
1493             fjy3             = _mm_macc_pd(dy33,fscal,fjy3);
1494             fjz3             = _mm_macc_pd(dz33,fscal,fjz3);
1495
1496             }
1497
1498             gmx_mm_decrement_4rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1499
1500             /* Inner loop uses 502 flops */
1501         }
1502
1503         /* End of innermost loop */
1504
1505         gmx_mm_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1506                                               f+i_coord_offset,fshift+i_shift_offset);
1507
1508         ggid                        = gid[iidx];
1509         /* Update potential energies */
1510         gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
1511         gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
1512
1513         /* Increment number of inner iterations */
1514         inneriter                  += j_index_end - j_index_start;
1515
1516         /* Outer loop uses 26 flops */
1517     }
1518
1519     /* Increment number of outer iterations */
1520     outeriter        += nri;
1521
1522     /* Update outer/inner flops */
1523
1524     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_VF,outeriter*26 + inneriter*502);
1525 }
1526 /*
1527  * Gromacs nonbonded kernel:   nb_kernel_ElecEwSh_VdwLJEwSh_GeomW4W4_F_avx_128_fma_double
1528  * Electrostatics interaction: Ewald
1529  * VdW interaction:            LJEwald
1530  * Geometry:                   Water4-Water4
1531  * Calculate force/pot:        Force
1532  */
1533 void
1534 nb_kernel_ElecEwSh_VdwLJEwSh_GeomW4W4_F_avx_128_fma_double
1535                     (t_nblist                    * gmx_restrict       nlist,
1536                      rvec                        * gmx_restrict          xx,
1537                      rvec                        * gmx_restrict          ff,
1538                      struct t_forcerec           * gmx_restrict          fr,
1539                      t_mdatoms                   * gmx_restrict     mdatoms,
1540                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1541                      t_nrnb                      * gmx_restrict        nrnb)
1542 {
1543     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1544      * just 0 for non-waters.
1545      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
1546      * jnr indices corresponding to data put in the four positions in the SIMD register.
1547      */
1548     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
1549     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1550     int              jnrA,jnrB;
1551     int              j_coord_offsetA,j_coord_offsetB;
1552     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
1553     real             rcutoff_scalar;
1554     real             *shiftvec,*fshift,*x,*f;
1555     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
1556     int              vdwioffset0;
1557     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1558     int              vdwioffset1;
1559     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1560     int              vdwioffset2;
1561     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1562     int              vdwioffset3;
1563     __m128d          ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
1564     int              vdwjidx0A,vdwjidx0B;
1565     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1566     int              vdwjidx1A,vdwjidx1B;
1567     __m128d          jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1568     int              vdwjidx2A,vdwjidx2B;
1569     __m128d          jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1570     int              vdwjidx3A,vdwjidx3B;
1571     __m128d          jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
1572     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1573     __m128d          dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1574     __m128d          dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1575     __m128d          dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
1576     __m128d          dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1577     __m128d          dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1578     __m128d          dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
1579     __m128d          dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
1580     __m128d          dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
1581     __m128d          dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
1582     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
1583     real             *charge;
1584     int              nvdwtype;
1585     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1586     int              *vdwtype;
1587     real             *vdwparam;
1588     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
1589     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
1590     __m128d           c6grid_00;
1591     __m128d           c6grid_11;
1592     __m128d           c6grid_12;
1593     __m128d           c6grid_13;
1594     __m128d           c6grid_21;
1595     __m128d           c6grid_22;
1596     __m128d           c6grid_23;
1597     __m128d           c6grid_31;
1598     __m128d           c6grid_32;
1599     __m128d           c6grid_33;
1600     real             *vdwgridparam;
1601     __m128d           ewclj,ewclj2,ewclj6,ewcljrsq,poly,exponent,f6A,f6B,sh_lj_ewald;
1602     __m128d           one_half  = _mm_set1_pd(0.5);
1603     __m128d           minus_one = _mm_set1_pd(-1.0);
1604     __m128i          ewitab;
1605     __m128d          ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
1606     real             *ewtab;
1607     __m128d          dummy_mask,cutoff_mask;
1608     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
1609     __m128d          one     = _mm_set1_pd(1.0);
1610     __m128d          two     = _mm_set1_pd(2.0);
1611     x                = xx[0];
1612     f                = ff[0];
1613
1614     nri              = nlist->nri;
1615     iinr             = nlist->iinr;
1616     jindex           = nlist->jindex;
1617     jjnr             = nlist->jjnr;
1618     shiftidx         = nlist->shift;
1619     gid              = nlist->gid;
1620     shiftvec         = fr->shift_vec[0];
1621     fshift           = fr->fshift[0];
1622     facel            = _mm_set1_pd(fr->ic->epsfac);
1623     charge           = mdatoms->chargeA;
1624     nvdwtype         = fr->ntype;
1625     vdwparam         = fr->nbfp;
1626     vdwtype          = mdatoms->typeA;
1627     vdwgridparam     = fr->ljpme_c6grid;
1628     sh_lj_ewald      = _mm_set1_pd(fr->ic->sh_lj_ewald);
1629     ewclj            = _mm_set1_pd(fr->ic->ewaldcoeff_lj);
1630     ewclj2           = _mm_mul_pd(minus_one,_mm_mul_pd(ewclj,ewclj));
1631
1632     sh_ewald         = _mm_set1_pd(fr->ic->sh_ewald);
1633     ewtab            = fr->ic->tabq_coul_F;
1634     ewtabscale       = _mm_set1_pd(fr->ic->tabq_scale);
1635     ewtabhalfspace   = _mm_set1_pd(0.5/fr->ic->tabq_scale);
1636
1637     /* Setup water-specific parameters */
1638     inr              = nlist->iinr[0];
1639     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
1640     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
1641     iq3              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
1642     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
1643
1644     jq1              = _mm_set1_pd(charge[inr+1]);
1645     jq2              = _mm_set1_pd(charge[inr+2]);
1646     jq3              = _mm_set1_pd(charge[inr+3]);
1647     vdwjidx0A        = 2*vdwtype[inr+0];
1648     c6_00            = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
1649     c12_00           = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
1650     c6grid_00        = _mm_set1_pd(vdwgridparam[vdwioffset0+vdwjidx0A]);
1651     qq11             = _mm_mul_pd(iq1,jq1);
1652     qq12             = _mm_mul_pd(iq1,jq2);
1653     qq13             = _mm_mul_pd(iq1,jq3);
1654     qq21             = _mm_mul_pd(iq2,jq1);
1655     qq22             = _mm_mul_pd(iq2,jq2);
1656     qq23             = _mm_mul_pd(iq2,jq3);
1657     qq31             = _mm_mul_pd(iq3,jq1);
1658     qq32             = _mm_mul_pd(iq3,jq2);
1659     qq33             = _mm_mul_pd(iq3,jq3);
1660
1661     /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
1662     rcutoff_scalar   = fr->ic->rcoulomb;
1663     rcutoff          = _mm_set1_pd(rcutoff_scalar);
1664     rcutoff2         = _mm_mul_pd(rcutoff,rcutoff);
1665
1666     sh_vdw_invrcut6  = _mm_set1_pd(fr->ic->sh_invrc6);
1667     rvdw             = _mm_set1_pd(fr->ic->rvdw);
1668
1669     /* Avoid stupid compiler warnings */
1670     jnrA = jnrB = 0;
1671     j_coord_offsetA = 0;
1672     j_coord_offsetB = 0;
1673
1674     outeriter        = 0;
1675     inneriter        = 0;
1676
1677     /* Start outer loop over neighborlists */
1678     for(iidx=0; iidx<nri; iidx++)
1679     {
1680         /* Load shift vector for this list */
1681         i_shift_offset   = DIM*shiftidx[iidx];
1682
1683         /* Load limits for loop over neighbors */
1684         j_index_start    = jindex[iidx];
1685         j_index_end      = jindex[iidx+1];
1686
1687         /* Get outer coordinate index */
1688         inr              = iinr[iidx];
1689         i_coord_offset   = DIM*inr;
1690
1691         /* Load i particle coords and add shift vector */
1692         gmx_mm_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
1693                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
1694
1695         fix0             = _mm_setzero_pd();
1696         fiy0             = _mm_setzero_pd();
1697         fiz0             = _mm_setzero_pd();
1698         fix1             = _mm_setzero_pd();
1699         fiy1             = _mm_setzero_pd();
1700         fiz1             = _mm_setzero_pd();
1701         fix2             = _mm_setzero_pd();
1702         fiy2             = _mm_setzero_pd();
1703         fiz2             = _mm_setzero_pd();
1704         fix3             = _mm_setzero_pd();
1705         fiy3             = _mm_setzero_pd();
1706         fiz3             = _mm_setzero_pd();
1707
1708         /* Start inner kernel loop */
1709         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
1710         {
1711
1712             /* Get j neighbor index, and coordinate index */
1713             jnrA             = jjnr[jidx];
1714             jnrB             = jjnr[jidx+1];
1715             j_coord_offsetA  = DIM*jnrA;
1716             j_coord_offsetB  = DIM*jnrB;
1717
1718             /* load j atom coordinates */
1719             gmx_mm_load_4rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1720                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1721                                               &jy2,&jz2,&jx3,&jy3,&jz3);
1722
1723             /* Calculate displacement vector */
1724             dx00             = _mm_sub_pd(ix0,jx0);
1725             dy00             = _mm_sub_pd(iy0,jy0);
1726             dz00             = _mm_sub_pd(iz0,jz0);
1727             dx11             = _mm_sub_pd(ix1,jx1);
1728             dy11             = _mm_sub_pd(iy1,jy1);
1729             dz11             = _mm_sub_pd(iz1,jz1);
1730             dx12             = _mm_sub_pd(ix1,jx2);
1731             dy12             = _mm_sub_pd(iy1,jy2);
1732             dz12             = _mm_sub_pd(iz1,jz2);
1733             dx13             = _mm_sub_pd(ix1,jx3);
1734             dy13             = _mm_sub_pd(iy1,jy3);
1735             dz13             = _mm_sub_pd(iz1,jz3);
1736             dx21             = _mm_sub_pd(ix2,jx1);
1737             dy21             = _mm_sub_pd(iy2,jy1);
1738             dz21             = _mm_sub_pd(iz2,jz1);
1739             dx22             = _mm_sub_pd(ix2,jx2);
1740             dy22             = _mm_sub_pd(iy2,jy2);
1741             dz22             = _mm_sub_pd(iz2,jz2);
1742             dx23             = _mm_sub_pd(ix2,jx3);
1743             dy23             = _mm_sub_pd(iy2,jy3);
1744             dz23             = _mm_sub_pd(iz2,jz3);
1745             dx31             = _mm_sub_pd(ix3,jx1);
1746             dy31             = _mm_sub_pd(iy3,jy1);
1747             dz31             = _mm_sub_pd(iz3,jz1);
1748             dx32             = _mm_sub_pd(ix3,jx2);
1749             dy32             = _mm_sub_pd(iy3,jy2);
1750             dz32             = _mm_sub_pd(iz3,jz2);
1751             dx33             = _mm_sub_pd(ix3,jx3);
1752             dy33             = _mm_sub_pd(iy3,jy3);
1753             dz33             = _mm_sub_pd(iz3,jz3);
1754
1755             /* Calculate squared distance and things based on it */
1756             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1757             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1758             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1759             rsq13            = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
1760             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1761             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1762             rsq23            = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
1763             rsq31            = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
1764             rsq32            = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
1765             rsq33            = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
1766
1767             rinv00           = avx128fma_invsqrt_d(rsq00);
1768             rinv11           = avx128fma_invsqrt_d(rsq11);
1769             rinv12           = avx128fma_invsqrt_d(rsq12);
1770             rinv13           = avx128fma_invsqrt_d(rsq13);
1771             rinv21           = avx128fma_invsqrt_d(rsq21);
1772             rinv22           = avx128fma_invsqrt_d(rsq22);
1773             rinv23           = avx128fma_invsqrt_d(rsq23);
1774             rinv31           = avx128fma_invsqrt_d(rsq31);
1775             rinv32           = avx128fma_invsqrt_d(rsq32);
1776             rinv33           = avx128fma_invsqrt_d(rsq33);
1777
1778             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
1779             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
1780             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
1781             rinvsq13         = _mm_mul_pd(rinv13,rinv13);
1782             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
1783             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
1784             rinvsq23         = _mm_mul_pd(rinv23,rinv23);
1785             rinvsq31         = _mm_mul_pd(rinv31,rinv31);
1786             rinvsq32         = _mm_mul_pd(rinv32,rinv32);
1787             rinvsq33         = _mm_mul_pd(rinv33,rinv33);
1788
1789             fjx0             = _mm_setzero_pd();
1790             fjy0             = _mm_setzero_pd();
1791             fjz0             = _mm_setzero_pd();
1792             fjx1             = _mm_setzero_pd();
1793             fjy1             = _mm_setzero_pd();
1794             fjz1             = _mm_setzero_pd();
1795             fjx2             = _mm_setzero_pd();
1796             fjy2             = _mm_setzero_pd();
1797             fjz2             = _mm_setzero_pd();
1798             fjx3             = _mm_setzero_pd();
1799             fjy3             = _mm_setzero_pd();
1800             fjz3             = _mm_setzero_pd();
1801
1802             /**************************
1803              * CALCULATE INTERACTIONS *
1804              **************************/
1805
1806             if (gmx_mm_any_lt(rsq00,rcutoff2))
1807             {
1808
1809             r00              = _mm_mul_pd(rsq00,rinv00);
1810
1811             /* Analytical LJ-PME */
1812             rinvsix          = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1813             ewcljrsq         = _mm_mul_pd(ewclj2,rsq00);
1814             ewclj6           = _mm_mul_pd(ewclj2,_mm_mul_pd(ewclj2,ewclj2));
1815             exponent         = avx128fma_exp_d(ewcljrsq);
1816             /* poly = exp(-(beta*r)^2) * (1 + (beta*r)^2 + (beta*r)^4 /2) */
1817             poly             = _mm_mul_pd(exponent,_mm_macc_pd(_mm_mul_pd(ewcljrsq,ewcljrsq),one_half,_mm_sub_pd(one,ewcljrsq)));
1818             /* f6A = 6 * C6grid * (1 - poly) */
1819             f6A              = _mm_mul_pd(c6grid_00,_mm_sub_pd(one,poly));
1820             /* f6B = C6grid * exponent * beta^6 */
1821             f6B              = _mm_mul_pd(_mm_mul_pd(c6grid_00,one_sixth),_mm_mul_pd(exponent,ewclj6));
1822             /* fvdw = 12*C12/r13 - ((6*C6 - f6A)/r6 + f6B)/r */
1823             fvdw              = _mm_mul_pd(_mm_macc_pd(_mm_msub_pd(c12_00,rinvsix,_mm_sub_pd(c6_00,f6A)),rinvsix,f6B),rinvsq00);
1824
1825             cutoff_mask      = _mm_cmplt_pd(rsq00,rcutoff2);
1826
1827             fscal            = fvdw;
1828
1829             fscal            = _mm_and_pd(fscal,cutoff_mask);
1830
1831             /* Update vectorial force */
1832             fix0             = _mm_macc_pd(dx00,fscal,fix0);
1833             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
1834             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
1835
1836             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
1837             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
1838             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
1839
1840             }
1841
1842             /**************************
1843              * CALCULATE INTERACTIONS *
1844              **************************/
1845
1846             if (gmx_mm_any_lt(rsq11,rcutoff2))
1847             {
1848
1849             r11              = _mm_mul_pd(rsq11,rinv11);
1850
1851             /* EWALD ELECTROSTATICS */
1852
1853             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1854             ewrt             = _mm_mul_pd(r11,ewtabscale);
1855             ewitab           = _mm_cvttpd_epi32(ewrt);
1856 #ifdef __XOP__
1857             eweps            = _mm_frcz_pd(ewrt);
1858 #else
1859             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1860 #endif
1861             twoeweps         = _mm_add_pd(eweps,eweps);
1862             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1863                                          &ewtabF,&ewtabFn);
1864             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1865             felec            = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
1866
1867             cutoff_mask      = _mm_cmplt_pd(rsq11,rcutoff2);
1868
1869             fscal            = felec;
1870
1871             fscal            = _mm_and_pd(fscal,cutoff_mask);
1872
1873             /* Update vectorial force */
1874             fix1             = _mm_macc_pd(dx11,fscal,fix1);
1875             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
1876             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
1877
1878             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
1879             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
1880             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
1881
1882             }
1883
1884             /**************************
1885              * CALCULATE INTERACTIONS *
1886              **************************/
1887
1888             if (gmx_mm_any_lt(rsq12,rcutoff2))
1889             {
1890
1891             r12              = _mm_mul_pd(rsq12,rinv12);
1892
1893             /* EWALD ELECTROSTATICS */
1894
1895             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1896             ewrt             = _mm_mul_pd(r12,ewtabscale);
1897             ewitab           = _mm_cvttpd_epi32(ewrt);
1898 #ifdef __XOP__
1899             eweps            = _mm_frcz_pd(ewrt);
1900 #else
1901             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1902 #endif
1903             twoeweps         = _mm_add_pd(eweps,eweps);
1904             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1905                                          &ewtabF,&ewtabFn);
1906             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1907             felec            = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
1908
1909             cutoff_mask      = _mm_cmplt_pd(rsq12,rcutoff2);
1910
1911             fscal            = felec;
1912
1913             fscal            = _mm_and_pd(fscal,cutoff_mask);
1914
1915             /* Update vectorial force */
1916             fix1             = _mm_macc_pd(dx12,fscal,fix1);
1917             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
1918             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
1919
1920             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
1921             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
1922             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
1923
1924             }
1925
1926             /**************************
1927              * CALCULATE INTERACTIONS *
1928              **************************/
1929
1930             if (gmx_mm_any_lt(rsq13,rcutoff2))
1931             {
1932
1933             r13              = _mm_mul_pd(rsq13,rinv13);
1934
1935             /* EWALD ELECTROSTATICS */
1936
1937             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1938             ewrt             = _mm_mul_pd(r13,ewtabscale);
1939             ewitab           = _mm_cvttpd_epi32(ewrt);
1940 #ifdef __XOP__
1941             eweps            = _mm_frcz_pd(ewrt);
1942 #else
1943             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1944 #endif
1945             twoeweps         = _mm_add_pd(eweps,eweps);
1946             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1947                                          &ewtabF,&ewtabFn);
1948             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1949             felec            = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
1950
1951             cutoff_mask      = _mm_cmplt_pd(rsq13,rcutoff2);
1952
1953             fscal            = felec;
1954
1955             fscal            = _mm_and_pd(fscal,cutoff_mask);
1956
1957             /* Update vectorial force */
1958             fix1             = _mm_macc_pd(dx13,fscal,fix1);
1959             fiy1             = _mm_macc_pd(dy13,fscal,fiy1);
1960             fiz1             = _mm_macc_pd(dz13,fscal,fiz1);
1961
1962             fjx3             = _mm_macc_pd(dx13,fscal,fjx3);
1963             fjy3             = _mm_macc_pd(dy13,fscal,fjy3);
1964             fjz3             = _mm_macc_pd(dz13,fscal,fjz3);
1965
1966             }
1967
1968             /**************************
1969              * CALCULATE INTERACTIONS *
1970              **************************/
1971
1972             if (gmx_mm_any_lt(rsq21,rcutoff2))
1973             {
1974
1975             r21              = _mm_mul_pd(rsq21,rinv21);
1976
1977             /* EWALD ELECTROSTATICS */
1978
1979             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1980             ewrt             = _mm_mul_pd(r21,ewtabscale);
1981             ewitab           = _mm_cvttpd_epi32(ewrt);
1982 #ifdef __XOP__
1983             eweps            = _mm_frcz_pd(ewrt);
1984 #else
1985             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1986 #endif
1987             twoeweps         = _mm_add_pd(eweps,eweps);
1988             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1989                                          &ewtabF,&ewtabFn);
1990             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1991             felec            = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
1992
1993             cutoff_mask      = _mm_cmplt_pd(rsq21,rcutoff2);
1994
1995             fscal            = felec;
1996
1997             fscal            = _mm_and_pd(fscal,cutoff_mask);
1998
1999             /* Update vectorial force */
2000             fix2             = _mm_macc_pd(dx21,fscal,fix2);
2001             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
2002             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
2003
2004             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
2005             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
2006             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
2007
2008             }
2009
2010             /**************************
2011              * CALCULATE INTERACTIONS *
2012              **************************/
2013
2014             if (gmx_mm_any_lt(rsq22,rcutoff2))
2015             {
2016
2017             r22              = _mm_mul_pd(rsq22,rinv22);
2018
2019             /* EWALD ELECTROSTATICS */
2020
2021             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2022             ewrt             = _mm_mul_pd(r22,ewtabscale);
2023             ewitab           = _mm_cvttpd_epi32(ewrt);
2024 #ifdef __XOP__
2025             eweps            = _mm_frcz_pd(ewrt);
2026 #else
2027             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2028 #endif
2029             twoeweps         = _mm_add_pd(eweps,eweps);
2030             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
2031                                          &ewtabF,&ewtabFn);
2032             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2033             felec            = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
2034
2035             cutoff_mask      = _mm_cmplt_pd(rsq22,rcutoff2);
2036
2037             fscal            = felec;
2038
2039             fscal            = _mm_and_pd(fscal,cutoff_mask);
2040
2041             /* Update vectorial force */
2042             fix2             = _mm_macc_pd(dx22,fscal,fix2);
2043             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
2044             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
2045
2046             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
2047             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
2048             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
2049
2050             }
2051
2052             /**************************
2053              * CALCULATE INTERACTIONS *
2054              **************************/
2055
2056             if (gmx_mm_any_lt(rsq23,rcutoff2))
2057             {
2058
2059             r23              = _mm_mul_pd(rsq23,rinv23);
2060
2061             /* EWALD ELECTROSTATICS */
2062
2063             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2064             ewrt             = _mm_mul_pd(r23,ewtabscale);
2065             ewitab           = _mm_cvttpd_epi32(ewrt);
2066 #ifdef __XOP__
2067             eweps            = _mm_frcz_pd(ewrt);
2068 #else
2069             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2070 #endif
2071             twoeweps         = _mm_add_pd(eweps,eweps);
2072             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
2073                                          &ewtabF,&ewtabFn);
2074             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2075             felec            = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
2076
2077             cutoff_mask      = _mm_cmplt_pd(rsq23,rcutoff2);
2078
2079             fscal            = felec;
2080
2081             fscal            = _mm_and_pd(fscal,cutoff_mask);
2082
2083             /* Update vectorial force */
2084             fix2             = _mm_macc_pd(dx23,fscal,fix2);
2085             fiy2             = _mm_macc_pd(dy23,fscal,fiy2);
2086             fiz2             = _mm_macc_pd(dz23,fscal,fiz2);
2087
2088             fjx3             = _mm_macc_pd(dx23,fscal,fjx3);
2089             fjy3             = _mm_macc_pd(dy23,fscal,fjy3);
2090             fjz3             = _mm_macc_pd(dz23,fscal,fjz3);
2091
2092             }
2093
2094             /**************************
2095              * CALCULATE INTERACTIONS *
2096              **************************/
2097
2098             if (gmx_mm_any_lt(rsq31,rcutoff2))
2099             {
2100
2101             r31              = _mm_mul_pd(rsq31,rinv31);
2102
2103             /* EWALD ELECTROSTATICS */
2104
2105             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2106             ewrt             = _mm_mul_pd(r31,ewtabscale);
2107             ewitab           = _mm_cvttpd_epi32(ewrt);
2108 #ifdef __XOP__
2109             eweps            = _mm_frcz_pd(ewrt);
2110 #else
2111             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2112 #endif
2113             twoeweps         = _mm_add_pd(eweps,eweps);
2114             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
2115                                          &ewtabF,&ewtabFn);
2116             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2117             felec            = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
2118
2119             cutoff_mask      = _mm_cmplt_pd(rsq31,rcutoff2);
2120
2121             fscal            = felec;
2122
2123             fscal            = _mm_and_pd(fscal,cutoff_mask);
2124
2125             /* Update vectorial force */
2126             fix3             = _mm_macc_pd(dx31,fscal,fix3);
2127             fiy3             = _mm_macc_pd(dy31,fscal,fiy3);
2128             fiz3             = _mm_macc_pd(dz31,fscal,fiz3);
2129
2130             fjx1             = _mm_macc_pd(dx31,fscal,fjx1);
2131             fjy1             = _mm_macc_pd(dy31,fscal,fjy1);
2132             fjz1             = _mm_macc_pd(dz31,fscal,fjz1);
2133
2134             }
2135
2136             /**************************
2137              * CALCULATE INTERACTIONS *
2138              **************************/
2139
2140             if (gmx_mm_any_lt(rsq32,rcutoff2))
2141             {
2142
2143             r32              = _mm_mul_pd(rsq32,rinv32);
2144
2145             /* EWALD ELECTROSTATICS */
2146
2147             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2148             ewrt             = _mm_mul_pd(r32,ewtabscale);
2149             ewitab           = _mm_cvttpd_epi32(ewrt);
2150 #ifdef __XOP__
2151             eweps            = _mm_frcz_pd(ewrt);
2152 #else
2153             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2154 #endif
2155             twoeweps         = _mm_add_pd(eweps,eweps);
2156             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
2157                                          &ewtabF,&ewtabFn);
2158             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2159             felec            = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
2160
2161             cutoff_mask      = _mm_cmplt_pd(rsq32,rcutoff2);
2162
2163             fscal            = felec;
2164
2165             fscal            = _mm_and_pd(fscal,cutoff_mask);
2166
2167             /* Update vectorial force */
2168             fix3             = _mm_macc_pd(dx32,fscal,fix3);
2169             fiy3             = _mm_macc_pd(dy32,fscal,fiy3);
2170             fiz3             = _mm_macc_pd(dz32,fscal,fiz3);
2171
2172             fjx2             = _mm_macc_pd(dx32,fscal,fjx2);
2173             fjy2             = _mm_macc_pd(dy32,fscal,fjy2);
2174             fjz2             = _mm_macc_pd(dz32,fscal,fjz2);
2175
2176             }
2177
2178             /**************************
2179              * CALCULATE INTERACTIONS *
2180              **************************/
2181
2182             if (gmx_mm_any_lt(rsq33,rcutoff2))
2183             {
2184
2185             r33              = _mm_mul_pd(rsq33,rinv33);
2186
2187             /* EWALD ELECTROSTATICS */
2188
2189             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2190             ewrt             = _mm_mul_pd(r33,ewtabscale);
2191             ewitab           = _mm_cvttpd_epi32(ewrt);
2192 #ifdef __XOP__
2193             eweps            = _mm_frcz_pd(ewrt);
2194 #else
2195             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2196 #endif
2197             twoeweps         = _mm_add_pd(eweps,eweps);
2198             gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
2199                                          &ewtabF,&ewtabFn);
2200             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2201             felec            = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
2202
2203             cutoff_mask      = _mm_cmplt_pd(rsq33,rcutoff2);
2204
2205             fscal            = felec;
2206
2207             fscal            = _mm_and_pd(fscal,cutoff_mask);
2208
2209             /* Update vectorial force */
2210             fix3             = _mm_macc_pd(dx33,fscal,fix3);
2211             fiy3             = _mm_macc_pd(dy33,fscal,fiy3);
2212             fiz3             = _mm_macc_pd(dz33,fscal,fiz3);
2213
2214             fjx3             = _mm_macc_pd(dx33,fscal,fjx3);
2215             fjy3             = _mm_macc_pd(dy33,fscal,fjy3);
2216             fjz3             = _mm_macc_pd(dz33,fscal,fjz3);
2217
2218             }
2219
2220             gmx_mm_decrement_4rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
2221
2222             /* Inner loop uses 431 flops */
2223         }
2224
2225         if(jidx<j_index_end)
2226         {
2227
2228             jnrA             = jjnr[jidx];
2229             j_coord_offsetA  = DIM*jnrA;
2230
2231             /* load j atom coordinates */
2232             gmx_mm_load_4rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
2233                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
2234                                               &jy2,&jz2,&jx3,&jy3,&jz3);
2235
2236             /* Calculate displacement vector */
2237             dx00             = _mm_sub_pd(ix0,jx0);
2238             dy00             = _mm_sub_pd(iy0,jy0);
2239             dz00             = _mm_sub_pd(iz0,jz0);
2240             dx11             = _mm_sub_pd(ix1,jx1);
2241             dy11             = _mm_sub_pd(iy1,jy1);
2242             dz11             = _mm_sub_pd(iz1,jz1);
2243             dx12             = _mm_sub_pd(ix1,jx2);
2244             dy12             = _mm_sub_pd(iy1,jy2);
2245             dz12             = _mm_sub_pd(iz1,jz2);
2246             dx13             = _mm_sub_pd(ix1,jx3);
2247             dy13             = _mm_sub_pd(iy1,jy3);
2248             dz13             = _mm_sub_pd(iz1,jz3);
2249             dx21             = _mm_sub_pd(ix2,jx1);
2250             dy21             = _mm_sub_pd(iy2,jy1);
2251             dz21             = _mm_sub_pd(iz2,jz1);
2252             dx22             = _mm_sub_pd(ix2,jx2);
2253             dy22             = _mm_sub_pd(iy2,jy2);
2254             dz22             = _mm_sub_pd(iz2,jz2);
2255             dx23             = _mm_sub_pd(ix2,jx3);
2256             dy23             = _mm_sub_pd(iy2,jy3);
2257             dz23             = _mm_sub_pd(iz2,jz3);
2258             dx31             = _mm_sub_pd(ix3,jx1);
2259             dy31             = _mm_sub_pd(iy3,jy1);
2260             dz31             = _mm_sub_pd(iz3,jz1);
2261             dx32             = _mm_sub_pd(ix3,jx2);
2262             dy32             = _mm_sub_pd(iy3,jy2);
2263             dz32             = _mm_sub_pd(iz3,jz2);
2264             dx33             = _mm_sub_pd(ix3,jx3);
2265             dy33             = _mm_sub_pd(iy3,jy3);
2266             dz33             = _mm_sub_pd(iz3,jz3);
2267
2268             /* Calculate squared distance and things based on it */
2269             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
2270             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
2271             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
2272             rsq13            = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
2273             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
2274             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
2275             rsq23            = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
2276             rsq31            = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
2277             rsq32            = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
2278             rsq33            = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
2279
2280             rinv00           = avx128fma_invsqrt_d(rsq00);
2281             rinv11           = avx128fma_invsqrt_d(rsq11);
2282             rinv12           = avx128fma_invsqrt_d(rsq12);
2283             rinv13           = avx128fma_invsqrt_d(rsq13);
2284             rinv21           = avx128fma_invsqrt_d(rsq21);
2285             rinv22           = avx128fma_invsqrt_d(rsq22);
2286             rinv23           = avx128fma_invsqrt_d(rsq23);
2287             rinv31           = avx128fma_invsqrt_d(rsq31);
2288             rinv32           = avx128fma_invsqrt_d(rsq32);
2289             rinv33           = avx128fma_invsqrt_d(rsq33);
2290
2291             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
2292             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
2293             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
2294             rinvsq13         = _mm_mul_pd(rinv13,rinv13);
2295             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
2296             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
2297             rinvsq23         = _mm_mul_pd(rinv23,rinv23);
2298             rinvsq31         = _mm_mul_pd(rinv31,rinv31);
2299             rinvsq32         = _mm_mul_pd(rinv32,rinv32);
2300             rinvsq33         = _mm_mul_pd(rinv33,rinv33);
2301
2302             fjx0             = _mm_setzero_pd();
2303             fjy0             = _mm_setzero_pd();
2304             fjz0             = _mm_setzero_pd();
2305             fjx1             = _mm_setzero_pd();
2306             fjy1             = _mm_setzero_pd();
2307             fjz1             = _mm_setzero_pd();
2308             fjx2             = _mm_setzero_pd();
2309             fjy2             = _mm_setzero_pd();
2310             fjz2             = _mm_setzero_pd();
2311             fjx3             = _mm_setzero_pd();
2312             fjy3             = _mm_setzero_pd();
2313             fjz3             = _mm_setzero_pd();
2314
2315             /**************************
2316              * CALCULATE INTERACTIONS *
2317              **************************/
2318
2319             if (gmx_mm_any_lt(rsq00,rcutoff2))
2320             {
2321
2322             r00              = _mm_mul_pd(rsq00,rinv00);
2323
2324             /* Analytical LJ-PME */
2325             rinvsix          = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
2326             ewcljrsq         = _mm_mul_pd(ewclj2,rsq00);
2327             ewclj6           = _mm_mul_pd(ewclj2,_mm_mul_pd(ewclj2,ewclj2));
2328             exponent         = avx128fma_exp_d(ewcljrsq);
2329             /* poly = exp(-(beta*r)^2) * (1 + (beta*r)^2 + (beta*r)^4 /2) */
2330             poly             = _mm_mul_pd(exponent,_mm_macc_pd(_mm_mul_pd(ewcljrsq,ewcljrsq),one_half,_mm_sub_pd(one,ewcljrsq)));
2331             /* f6A = 6 * C6grid * (1 - poly) */
2332             f6A              = _mm_mul_pd(c6grid_00,_mm_sub_pd(one,poly));
2333             /* f6B = C6grid * exponent * beta^6 */
2334             f6B              = _mm_mul_pd(_mm_mul_pd(c6grid_00,one_sixth),_mm_mul_pd(exponent,ewclj6));
2335             /* fvdw = 12*C12/r13 - ((6*C6 - f6A)/r6 + f6B)/r */
2336             fvdw              = _mm_mul_pd(_mm_macc_pd(_mm_msub_pd(c12_00,rinvsix,_mm_sub_pd(c6_00,f6A)),rinvsix,f6B),rinvsq00);
2337
2338             cutoff_mask      = _mm_cmplt_pd(rsq00,rcutoff2);
2339
2340             fscal            = fvdw;
2341
2342             fscal            = _mm_and_pd(fscal,cutoff_mask);
2343
2344             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2345
2346             /* Update vectorial force */
2347             fix0             = _mm_macc_pd(dx00,fscal,fix0);
2348             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
2349             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
2350
2351             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
2352             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
2353             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
2354
2355             }
2356
2357             /**************************
2358              * CALCULATE INTERACTIONS *
2359              **************************/
2360
2361             if (gmx_mm_any_lt(rsq11,rcutoff2))
2362             {
2363
2364             r11              = _mm_mul_pd(rsq11,rinv11);
2365
2366             /* EWALD ELECTROSTATICS */
2367
2368             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2369             ewrt             = _mm_mul_pd(r11,ewtabscale);
2370             ewitab           = _mm_cvttpd_epi32(ewrt);
2371 #ifdef __XOP__
2372             eweps            = _mm_frcz_pd(ewrt);
2373 #else
2374             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2375 #endif
2376             twoeweps         = _mm_add_pd(eweps,eweps);
2377             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2378             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2379             felec            = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
2380
2381             cutoff_mask      = _mm_cmplt_pd(rsq11,rcutoff2);
2382
2383             fscal            = felec;
2384
2385             fscal            = _mm_and_pd(fscal,cutoff_mask);
2386
2387             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2388
2389             /* Update vectorial force */
2390             fix1             = _mm_macc_pd(dx11,fscal,fix1);
2391             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
2392             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
2393
2394             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
2395             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
2396             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
2397
2398             }
2399
2400             /**************************
2401              * CALCULATE INTERACTIONS *
2402              **************************/
2403
2404             if (gmx_mm_any_lt(rsq12,rcutoff2))
2405             {
2406
2407             r12              = _mm_mul_pd(rsq12,rinv12);
2408
2409             /* EWALD ELECTROSTATICS */
2410
2411             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2412             ewrt             = _mm_mul_pd(r12,ewtabscale);
2413             ewitab           = _mm_cvttpd_epi32(ewrt);
2414 #ifdef __XOP__
2415             eweps            = _mm_frcz_pd(ewrt);
2416 #else
2417             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2418 #endif
2419             twoeweps         = _mm_add_pd(eweps,eweps);
2420             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2421             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2422             felec            = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
2423
2424             cutoff_mask      = _mm_cmplt_pd(rsq12,rcutoff2);
2425
2426             fscal            = felec;
2427
2428             fscal            = _mm_and_pd(fscal,cutoff_mask);
2429
2430             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2431
2432             /* Update vectorial force */
2433             fix1             = _mm_macc_pd(dx12,fscal,fix1);
2434             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
2435             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
2436
2437             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
2438             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
2439             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
2440
2441             }
2442
2443             /**************************
2444              * CALCULATE INTERACTIONS *
2445              **************************/
2446
2447             if (gmx_mm_any_lt(rsq13,rcutoff2))
2448             {
2449
2450             r13              = _mm_mul_pd(rsq13,rinv13);
2451
2452             /* EWALD ELECTROSTATICS */
2453
2454             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2455             ewrt             = _mm_mul_pd(r13,ewtabscale);
2456             ewitab           = _mm_cvttpd_epi32(ewrt);
2457 #ifdef __XOP__
2458             eweps            = _mm_frcz_pd(ewrt);
2459 #else
2460             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2461 #endif
2462             twoeweps         = _mm_add_pd(eweps,eweps);
2463             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2464             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2465             felec            = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
2466
2467             cutoff_mask      = _mm_cmplt_pd(rsq13,rcutoff2);
2468
2469             fscal            = felec;
2470
2471             fscal            = _mm_and_pd(fscal,cutoff_mask);
2472
2473             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2474
2475             /* Update vectorial force */
2476             fix1             = _mm_macc_pd(dx13,fscal,fix1);
2477             fiy1             = _mm_macc_pd(dy13,fscal,fiy1);
2478             fiz1             = _mm_macc_pd(dz13,fscal,fiz1);
2479
2480             fjx3             = _mm_macc_pd(dx13,fscal,fjx3);
2481             fjy3             = _mm_macc_pd(dy13,fscal,fjy3);
2482             fjz3             = _mm_macc_pd(dz13,fscal,fjz3);
2483
2484             }
2485
2486             /**************************
2487              * CALCULATE INTERACTIONS *
2488              **************************/
2489
2490             if (gmx_mm_any_lt(rsq21,rcutoff2))
2491             {
2492
2493             r21              = _mm_mul_pd(rsq21,rinv21);
2494
2495             /* EWALD ELECTROSTATICS */
2496
2497             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2498             ewrt             = _mm_mul_pd(r21,ewtabscale);
2499             ewitab           = _mm_cvttpd_epi32(ewrt);
2500 #ifdef __XOP__
2501             eweps            = _mm_frcz_pd(ewrt);
2502 #else
2503             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2504 #endif
2505             twoeweps         = _mm_add_pd(eweps,eweps);
2506             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2507             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2508             felec            = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
2509
2510             cutoff_mask      = _mm_cmplt_pd(rsq21,rcutoff2);
2511
2512             fscal            = felec;
2513
2514             fscal            = _mm_and_pd(fscal,cutoff_mask);
2515
2516             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2517
2518             /* Update vectorial force */
2519             fix2             = _mm_macc_pd(dx21,fscal,fix2);
2520             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
2521             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
2522
2523             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
2524             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
2525             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
2526
2527             }
2528
2529             /**************************
2530              * CALCULATE INTERACTIONS *
2531              **************************/
2532
2533             if (gmx_mm_any_lt(rsq22,rcutoff2))
2534             {
2535
2536             r22              = _mm_mul_pd(rsq22,rinv22);
2537
2538             /* EWALD ELECTROSTATICS */
2539
2540             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2541             ewrt             = _mm_mul_pd(r22,ewtabscale);
2542             ewitab           = _mm_cvttpd_epi32(ewrt);
2543 #ifdef __XOP__
2544             eweps            = _mm_frcz_pd(ewrt);
2545 #else
2546             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2547 #endif
2548             twoeweps         = _mm_add_pd(eweps,eweps);
2549             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2550             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2551             felec            = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
2552
2553             cutoff_mask      = _mm_cmplt_pd(rsq22,rcutoff2);
2554
2555             fscal            = felec;
2556
2557             fscal            = _mm_and_pd(fscal,cutoff_mask);
2558
2559             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2560
2561             /* Update vectorial force */
2562             fix2             = _mm_macc_pd(dx22,fscal,fix2);
2563             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
2564             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
2565
2566             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
2567             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
2568             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
2569
2570             }
2571
2572             /**************************
2573              * CALCULATE INTERACTIONS *
2574              **************************/
2575
2576             if (gmx_mm_any_lt(rsq23,rcutoff2))
2577             {
2578
2579             r23              = _mm_mul_pd(rsq23,rinv23);
2580
2581             /* EWALD ELECTROSTATICS */
2582
2583             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2584             ewrt             = _mm_mul_pd(r23,ewtabscale);
2585             ewitab           = _mm_cvttpd_epi32(ewrt);
2586 #ifdef __XOP__
2587             eweps            = _mm_frcz_pd(ewrt);
2588 #else
2589             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2590 #endif
2591             twoeweps         = _mm_add_pd(eweps,eweps);
2592             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2593             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2594             felec            = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
2595
2596             cutoff_mask      = _mm_cmplt_pd(rsq23,rcutoff2);
2597
2598             fscal            = felec;
2599
2600             fscal            = _mm_and_pd(fscal,cutoff_mask);
2601
2602             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2603
2604             /* Update vectorial force */
2605             fix2             = _mm_macc_pd(dx23,fscal,fix2);
2606             fiy2             = _mm_macc_pd(dy23,fscal,fiy2);
2607             fiz2             = _mm_macc_pd(dz23,fscal,fiz2);
2608
2609             fjx3             = _mm_macc_pd(dx23,fscal,fjx3);
2610             fjy3             = _mm_macc_pd(dy23,fscal,fjy3);
2611             fjz3             = _mm_macc_pd(dz23,fscal,fjz3);
2612
2613             }
2614
2615             /**************************
2616              * CALCULATE INTERACTIONS *
2617              **************************/
2618
2619             if (gmx_mm_any_lt(rsq31,rcutoff2))
2620             {
2621
2622             r31              = _mm_mul_pd(rsq31,rinv31);
2623
2624             /* EWALD ELECTROSTATICS */
2625
2626             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2627             ewrt             = _mm_mul_pd(r31,ewtabscale);
2628             ewitab           = _mm_cvttpd_epi32(ewrt);
2629 #ifdef __XOP__
2630             eweps            = _mm_frcz_pd(ewrt);
2631 #else
2632             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2633 #endif
2634             twoeweps         = _mm_add_pd(eweps,eweps);
2635             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2636             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2637             felec            = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
2638
2639             cutoff_mask      = _mm_cmplt_pd(rsq31,rcutoff2);
2640
2641             fscal            = felec;
2642
2643             fscal            = _mm_and_pd(fscal,cutoff_mask);
2644
2645             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2646
2647             /* Update vectorial force */
2648             fix3             = _mm_macc_pd(dx31,fscal,fix3);
2649             fiy3             = _mm_macc_pd(dy31,fscal,fiy3);
2650             fiz3             = _mm_macc_pd(dz31,fscal,fiz3);
2651
2652             fjx1             = _mm_macc_pd(dx31,fscal,fjx1);
2653             fjy1             = _mm_macc_pd(dy31,fscal,fjy1);
2654             fjz1             = _mm_macc_pd(dz31,fscal,fjz1);
2655
2656             }
2657
2658             /**************************
2659              * CALCULATE INTERACTIONS *
2660              **************************/
2661
2662             if (gmx_mm_any_lt(rsq32,rcutoff2))
2663             {
2664
2665             r32              = _mm_mul_pd(rsq32,rinv32);
2666
2667             /* EWALD ELECTROSTATICS */
2668
2669             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2670             ewrt             = _mm_mul_pd(r32,ewtabscale);
2671             ewitab           = _mm_cvttpd_epi32(ewrt);
2672 #ifdef __XOP__
2673             eweps            = _mm_frcz_pd(ewrt);
2674 #else
2675             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2676 #endif
2677             twoeweps         = _mm_add_pd(eweps,eweps);
2678             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2679             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2680             felec            = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
2681
2682             cutoff_mask      = _mm_cmplt_pd(rsq32,rcutoff2);
2683
2684             fscal            = felec;
2685
2686             fscal            = _mm_and_pd(fscal,cutoff_mask);
2687
2688             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2689
2690             /* Update vectorial force */
2691             fix3             = _mm_macc_pd(dx32,fscal,fix3);
2692             fiy3             = _mm_macc_pd(dy32,fscal,fiy3);
2693             fiz3             = _mm_macc_pd(dz32,fscal,fiz3);
2694
2695             fjx2             = _mm_macc_pd(dx32,fscal,fjx2);
2696             fjy2             = _mm_macc_pd(dy32,fscal,fjy2);
2697             fjz2             = _mm_macc_pd(dz32,fscal,fjz2);
2698
2699             }
2700
2701             /**************************
2702              * CALCULATE INTERACTIONS *
2703              **************************/
2704
2705             if (gmx_mm_any_lt(rsq33,rcutoff2))
2706             {
2707
2708             r33              = _mm_mul_pd(rsq33,rinv33);
2709
2710             /* EWALD ELECTROSTATICS */
2711
2712             /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2713             ewrt             = _mm_mul_pd(r33,ewtabscale);
2714             ewitab           = _mm_cvttpd_epi32(ewrt);
2715 #ifdef __XOP__
2716             eweps            = _mm_frcz_pd(ewrt);
2717 #else
2718             eweps            = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2719 #endif
2720             twoeweps         = _mm_add_pd(eweps,eweps);
2721             gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2722             felec            = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2723             felec            = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
2724
2725             cutoff_mask      = _mm_cmplt_pd(rsq33,rcutoff2);
2726
2727             fscal            = felec;
2728
2729             fscal            = _mm_and_pd(fscal,cutoff_mask);
2730
2731             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2732
2733             /* Update vectorial force */
2734             fix3             = _mm_macc_pd(dx33,fscal,fix3);
2735             fiy3             = _mm_macc_pd(dy33,fscal,fiy3);
2736             fiz3             = _mm_macc_pd(dz33,fscal,fiz3);
2737
2738             fjx3             = _mm_macc_pd(dx33,fscal,fjx3);
2739             fjy3             = _mm_macc_pd(dy33,fscal,fjy3);
2740             fjz3             = _mm_macc_pd(dz33,fscal,fjz3);
2741
2742             }
2743
2744             gmx_mm_decrement_4rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
2745
2746             /* Inner loop uses 431 flops */
2747         }
2748
2749         /* End of innermost loop */
2750
2751         gmx_mm_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
2752                                               f+i_coord_offset,fshift+i_shift_offset);
2753
2754         /* Increment number of inner iterations */
2755         inneriter                  += j_index_end - j_index_start;
2756
2757         /* Outer loop uses 24 flops */
2758     }
2759
2760     /* Increment number of outer iterations */
2761     outeriter        += nri;
2762
2763     /* Update outer/inner flops */
2764
2765     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_F,outeriter*24 + inneriter*431);
2766 }