src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_double/nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_avx_128_fma_double.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_double kernel generator.
  37  */
  38 #include "config.h"
  39
  40 #include <math.h>
  41
  42 #include "../nb_kernel.h"
  43 #include "types/simple.h"
  44 #include "gromacs/math/vec.h"
  45 #include "nrnb.h"
  46
  47 #include "gromacs/simd/math_x86_avx_128_fma_double.h"
  48 #include "kernelutil_x86_avx_128_fma_double.h"
  49
  50 /*
  51  * Gromacs nonbonded kernel:   nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_VF_avx_128_fma_double
  52  * Electrostatics interaction: Coulomb
  53  * VdW interaction:            CubicSplineTable
  54  * Geometry:                   Water3-Water3
  55  * Calculate force/pot:        PotentialAndForce
  56  */
  57 void
  58 nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_VF_avx_128_fma_double
  59                     (t_nblist                    * gmx_restrict       nlist,
  60                      rvec                        * gmx_restrict          xx,
  61                      rvec                        * gmx_restrict          ff,
  62                      t_forcerec                  * gmx_restrict          fr,
  63                      t_mdatoms                   * gmx_restrict     mdatoms,
  64                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  65                      t_nrnb                      * gmx_restrict        nrnb)
  66 {
  67     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  68      * just 0 for non-waters.
  69      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
  70      * jnr indices corresponding to data put in the four positions in the SIMD register.
  71      */
  72     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  73     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  74     int              jnrA,jnrB;
  75     int              j_coord_offsetA,j_coord_offsetB;
  76     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  77     real             rcutoff_scalar;
  78     real             *shiftvec,*fshift,*x,*f;
  79     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
  80     int              vdwioffset0;
  81     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  82     int              vdwioffset1;
  83     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  84     int              vdwioffset2;
  85     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  86     int              vdwjidx0A,vdwjidx0B;
  87     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  88     int              vdwjidx1A,vdwjidx1B;
  89     __m128d          jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  90     int              vdwjidx2A,vdwjidx2B;
  91     __m128d          jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  92     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  93     __m128d          dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
  94     __m128d          dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
  95     __m128d          dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  96     __m128d          dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
  97     __m128d          dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
  98     __m128d          dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
  99     __m128d          dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 100     __m128d          dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 101     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
 102     real             *charge;
 103     int              nvdwtype;
 104     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 105     int              *vdwtype;
 106     real             *vdwparam;
 107     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
 108     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
 109     __m128i          vfitab;
 110     __m128i          ifour       = _mm_set1_epi32(4);
 111     __m128d          rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF,twovfeps;
 112     real             *vftab;
 113     __m128d          dummy_mask,cutoff_mask;
 114     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
 115     __m128d          one     = _mm_set1_pd(1.0);
 116     __m128d          two     = _mm_set1_pd(2.0);
 117     x                = xx[0];
 118     f                = ff[0];
 119
 120     nri              = nlist->nri;
 121     iinr             = nlist->iinr;
 122     jindex           = nlist->jindex;
 123     jjnr             = nlist->jjnr;
 124     shiftidx         = nlist->shift;
 125     gid              = nlist->gid;
 126     shiftvec         = fr->shift_vec[0];
 127     fshift           = fr->fshift[0];
 128     facel            = _mm_set1_pd(fr->epsfac);
 129     charge           = mdatoms->chargeA;
 130     nvdwtype         = fr->ntype;
 131     vdwparam         = fr->nbfp;
 132     vdwtype          = mdatoms->typeA;
 133
 134     vftab            = kernel_data->table_vdw->data;
 135     vftabscale       = _mm_set1_pd(kernel_data->table_vdw->scale);
 136
 137     /* Setup water-specific parameters */
 138     inr              = nlist->iinr[0];
 139     iq0              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
 140     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
 141     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
 142     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 143
 144     jq0              = _mm_set1_pd(charge[inr+0]);
 145     jq1              = _mm_set1_pd(charge[inr+1]);
 146     jq2              = _mm_set1_pd(charge[inr+2]);
 147     vdwjidx0A        = 2*vdwtype[inr+0];
 148     qq00             = _mm_mul_pd(iq0,jq0);
 149     c6_00            = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
 150     c12_00           = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
 151     qq01             = _mm_mul_pd(iq0,jq1);
 152     qq02             = _mm_mul_pd(iq0,jq2);
 153     qq10             = _mm_mul_pd(iq1,jq0);
 154     qq11             = _mm_mul_pd(iq1,jq1);
 155     qq12             = _mm_mul_pd(iq1,jq2);
 156     qq20             = _mm_mul_pd(iq2,jq0);
 157     qq21             = _mm_mul_pd(iq2,jq1);
 158     qq22             = _mm_mul_pd(iq2,jq2);
 159
 160     /* Avoid stupid compiler warnings */
 161     jnrA = jnrB = 0;
 162     j_coord_offsetA = 0;
 163     j_coord_offsetB = 0;
 164
 165     outeriter        = 0;
 166     inneriter        = 0;
 167
 168     /* Start outer loop over neighborlists */
 169     for(iidx=0; iidx<nri; iidx++)
 170     {
 171         /* Load shift vector for this list */
 172         i_shift_offset   = DIM*shiftidx[iidx];
 173
 174         /* Load limits for loop over neighbors */
 175         j_index_start    = jindex[iidx];
 176         j_index_end      = jindex[iidx+1];
 177
 178         /* Get outer coordinate index */
 179         inr              = iinr[iidx];
 180         i_coord_offset   = DIM*inr;
 181
 182         /* Load i particle coords and add shift vector */
 183         gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
 184                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 185
 186         fix0             = _mm_setzero_pd();
 187         fiy0             = _mm_setzero_pd();
 188         fiz0             = _mm_setzero_pd();
 189         fix1             = _mm_setzero_pd();
 190         fiy1             = _mm_setzero_pd();
 191         fiz1             = _mm_setzero_pd();
 192         fix2             = _mm_setzero_pd();
 193         fiy2             = _mm_setzero_pd();
 194         fiz2             = _mm_setzero_pd();
 195
 196         /* Reset potential sums */
 197         velecsum         = _mm_setzero_pd();
 198         vvdwsum          = _mm_setzero_pd();
 199
 200         /* Start inner kernel loop */
 201         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
 202         {
 203
 204             /* Get j neighbor index, and coordinate index */
 205             jnrA             = jjnr[jidx];
 206             jnrB             = jjnr[jidx+1];
 207             j_coord_offsetA  = DIM*jnrA;
 208             j_coord_offsetB  = DIM*jnrB;
 209
 210             /* load j atom coordinates */
 211             gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
 212                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 213
 214             /* Calculate displacement vector */
 215             dx00             = _mm_sub_pd(ix0,jx0);
 216             dy00             = _mm_sub_pd(iy0,jy0);
 217             dz00             = _mm_sub_pd(iz0,jz0);
 218             dx01             = _mm_sub_pd(ix0,jx1);
 219             dy01             = _mm_sub_pd(iy0,jy1);
 220             dz01             = _mm_sub_pd(iz0,jz1);
 221             dx02             = _mm_sub_pd(ix0,jx2);
 222             dy02             = _mm_sub_pd(iy0,jy2);
 223             dz02             = _mm_sub_pd(iz0,jz2);
 224             dx10             = _mm_sub_pd(ix1,jx0);
 225             dy10             = _mm_sub_pd(iy1,jy0);
 226             dz10             = _mm_sub_pd(iz1,jz0);
 227             dx11             = _mm_sub_pd(ix1,jx1);
 228             dy11             = _mm_sub_pd(iy1,jy1);
 229             dz11             = _mm_sub_pd(iz1,jz1);
 230             dx12             = _mm_sub_pd(ix1,jx2);
 231             dy12             = _mm_sub_pd(iy1,jy2);
 232             dz12             = _mm_sub_pd(iz1,jz2);
 233             dx20             = _mm_sub_pd(ix2,jx0);
 234             dy20             = _mm_sub_pd(iy2,jy0);
 235             dz20             = _mm_sub_pd(iz2,jz0);
 236             dx21             = _mm_sub_pd(ix2,jx1);
 237             dy21             = _mm_sub_pd(iy2,jy1);
 238             dz21             = _mm_sub_pd(iz2,jz1);
 239             dx22             = _mm_sub_pd(ix2,jx2);
 240             dy22             = _mm_sub_pd(iy2,jy2);
 241             dz22             = _mm_sub_pd(iz2,jz2);
 242
 243             /* Calculate squared distance and things based on it */
 244             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 245             rsq01            = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
 246             rsq02            = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
 247             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 248             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
 249             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
 250             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 251             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
 252             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
 253
 254             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 255             rinv01           = gmx_mm_invsqrt_pd(rsq01);
 256             rinv02           = gmx_mm_invsqrt_pd(rsq02);
 257             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 258             rinv11           = gmx_mm_invsqrt_pd(rsq11);
 259             rinv12           = gmx_mm_invsqrt_pd(rsq12);
 260             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 261             rinv21           = gmx_mm_invsqrt_pd(rsq21);
 262             rinv22           = gmx_mm_invsqrt_pd(rsq22);
 263
 264             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
 265             rinvsq01         = _mm_mul_pd(rinv01,rinv01);
 266             rinvsq02         = _mm_mul_pd(rinv02,rinv02);
 267             rinvsq10         = _mm_mul_pd(rinv10,rinv10);
 268             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
 269             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
 270             rinvsq20         = _mm_mul_pd(rinv20,rinv20);
 271             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
 272             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
 273
 274             fjx0             = _mm_setzero_pd();
 275             fjy0             = _mm_setzero_pd();
 276             fjz0             = _mm_setzero_pd();
 277             fjx1             = _mm_setzero_pd();
 278             fjy1             = _mm_setzero_pd();
 279             fjz1             = _mm_setzero_pd();
 280             fjx2             = _mm_setzero_pd();
 281             fjy2             = _mm_setzero_pd();
 282             fjz2             = _mm_setzero_pd();
 283
 284             /**************************
 285              * CALCULATE INTERACTIONS *
 286              **************************/
 287
 288             r00              = _mm_mul_pd(rsq00,rinv00);
 289
 290             /* Calculate table index by multiplying r with table scale and truncate to integer */
 291             rt               = _mm_mul_pd(r00,vftabscale);
 292             vfitab           = _mm_cvttpd_epi32(rt);
 293 #ifdef __XOP__
 294             vfeps            = _mm_frcz_pd(rt);
 295 #else
 296             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 297 #endif
 298             twovfeps         = _mm_add_pd(vfeps,vfeps);
 299             vfitab           = _mm_slli_epi32(vfitab,3);
 300
 301             /* COULOMB ELECTROSTATICS */
 302             velec            = _mm_mul_pd(qq00,rinv00);
 303             felec            = _mm_mul_pd(velec,rinvsq00);
 304
 305             /* CUBIC SPLINE TABLE DISPERSION */
 306             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
 307             F                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
 308             GMX_MM_TRANSPOSE2_PD(Y,F);
 309             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
 310             H                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
 311             GMX_MM_TRANSPOSE2_PD(G,H);
 312             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
 313             VV               = _mm_macc_pd(vfeps,Fp,Y);
 314             vvdw6            = _mm_mul_pd(c6_00,VV);
 315             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
 316             fvdw6            = _mm_mul_pd(c6_00,FF);
 317
 318             /* CUBIC SPLINE TABLE REPULSION */
 319             vfitab           = _mm_add_epi32(vfitab,ifour);
 320             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
 321             F                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
 322             GMX_MM_TRANSPOSE2_PD(Y,F);
 323             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
 324             H                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
 325             GMX_MM_TRANSPOSE2_PD(G,H);
 326             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
 327             VV               = _mm_macc_pd(vfeps,Fp,Y);
 328             vvdw12           = _mm_mul_pd(c12_00,VV);
 329             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
 330             fvdw12           = _mm_mul_pd(c12_00,FF);
 331             vvdw             = _mm_add_pd(vvdw12,vvdw6);
 332             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
 333
 334             /* Update potential sum for this i atom from the interaction with this j atom. */
 335             velecsum         = _mm_add_pd(velecsum,velec);
 336             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 337
 338             fscal            = _mm_add_pd(felec,fvdw);
 339
 340             /* Update vectorial force */
 341             fix0             = _mm_macc_pd(dx00,fscal,fix0);
 342             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
 343             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
 344
 345             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
 346             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
 347             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
 348
 349             /**************************
 350              * CALCULATE INTERACTIONS *
 351              **************************/
 352
 353             /* COULOMB ELECTROSTATICS */
 354             velec            = _mm_mul_pd(qq01,rinv01);
 355             felec            = _mm_mul_pd(velec,rinvsq01);
 356
 357             /* Update potential sum for this i atom from the interaction with this j atom. */
 358             velecsum         = _mm_add_pd(velecsum,velec);
 359
 360             fscal            = felec;
 361
 362             /* Update vectorial force */
 363             fix0             = _mm_macc_pd(dx01,fscal,fix0);
 364             fiy0             = _mm_macc_pd(dy01,fscal,fiy0);
 365             fiz0             = _mm_macc_pd(dz01,fscal,fiz0);
 366
 367             fjx1             = _mm_macc_pd(dx01,fscal,fjx1);
 368             fjy1             = _mm_macc_pd(dy01,fscal,fjy1);
 369             fjz1             = _mm_macc_pd(dz01,fscal,fjz1);
 370
 371             /**************************
 372              * CALCULATE INTERACTIONS *
 373              **************************/
 374
 375             /* COULOMB ELECTROSTATICS */
 376             velec            = _mm_mul_pd(qq02,rinv02);
 377             felec            = _mm_mul_pd(velec,rinvsq02);
 378
 379             /* Update potential sum for this i atom from the interaction with this j atom. */
 380             velecsum         = _mm_add_pd(velecsum,velec);
 381
 382             fscal            = felec;
 383
 384             /* Update vectorial force */
 385             fix0             = _mm_macc_pd(dx02,fscal,fix0);
 386             fiy0             = _mm_macc_pd(dy02,fscal,fiy0);
 387             fiz0             = _mm_macc_pd(dz02,fscal,fiz0);
 388
 389             fjx2             = _mm_macc_pd(dx02,fscal,fjx2);
 390             fjy2             = _mm_macc_pd(dy02,fscal,fjy2);
 391             fjz2             = _mm_macc_pd(dz02,fscal,fjz2);
 392
 393             /**************************
 394              * CALCULATE INTERACTIONS *
 395              **************************/
 396
 397             /* COULOMB ELECTROSTATICS */
 398             velec            = _mm_mul_pd(qq10,rinv10);
 399             felec            = _mm_mul_pd(velec,rinvsq10);
 400
 401             /* Update potential sum for this i atom from the interaction with this j atom. */
 402             velecsum         = _mm_add_pd(velecsum,velec);
 403
 404             fscal            = felec;
 405
 406             /* Update vectorial force */
 407             fix1             = _mm_macc_pd(dx10,fscal,fix1);
 408             fiy1             = _mm_macc_pd(dy10,fscal,fiy1);
 409             fiz1             = _mm_macc_pd(dz10,fscal,fiz1);
 410
 411             fjx0             = _mm_macc_pd(dx10,fscal,fjx0);
 412             fjy0             = _mm_macc_pd(dy10,fscal,fjy0);
 413             fjz0             = _mm_macc_pd(dz10,fscal,fjz0);
 414
 415             /**************************
 416              * CALCULATE INTERACTIONS *
 417              **************************/
 418
 419             /* COULOMB ELECTROSTATICS */
 420             velec            = _mm_mul_pd(qq11,rinv11);
 421             felec            = _mm_mul_pd(velec,rinvsq11);
 422
 423             /* Update potential sum for this i atom from the interaction with this j atom. */
 424             velecsum         = _mm_add_pd(velecsum,velec);
 425
 426             fscal            = felec;
 427
 428             /* Update vectorial force */
 429             fix1             = _mm_macc_pd(dx11,fscal,fix1);
 430             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
 431             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
 432
 433             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
 434             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
 435             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
 436
 437             /**************************
 438              * CALCULATE INTERACTIONS *
 439              **************************/
 440
 441             /* COULOMB ELECTROSTATICS */
 442             velec            = _mm_mul_pd(qq12,rinv12);
 443             felec            = _mm_mul_pd(velec,rinvsq12);
 444
 445             /* Update potential sum for this i atom from the interaction with this j atom. */
 446             velecsum         = _mm_add_pd(velecsum,velec);
 447
 448             fscal            = felec;
 449
 450             /* Update vectorial force */
 451             fix1             = _mm_macc_pd(dx12,fscal,fix1);
 452             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
 453             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
 454
 455             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
 456             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
 457             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
 458
 459             /**************************
 460              * CALCULATE INTERACTIONS *
 461              **************************/
 462
 463             /* COULOMB ELECTROSTATICS */
 464             velec            = _mm_mul_pd(qq20,rinv20);
 465             felec            = _mm_mul_pd(velec,rinvsq20);
 466
 467             /* Update potential sum for this i atom from the interaction with this j atom. */
 468             velecsum         = _mm_add_pd(velecsum,velec);
 469
 470             fscal            = felec;
 471
 472             /* Update vectorial force */
 473             fix2             = _mm_macc_pd(dx20,fscal,fix2);
 474             fiy2             = _mm_macc_pd(dy20,fscal,fiy2);
 475             fiz2             = _mm_macc_pd(dz20,fscal,fiz2);
 476
 477             fjx0             = _mm_macc_pd(dx20,fscal,fjx0);
 478             fjy0             = _mm_macc_pd(dy20,fscal,fjy0);
 479             fjz0             = _mm_macc_pd(dz20,fscal,fjz0);
 480
 481             /**************************
 482              * CALCULATE INTERACTIONS *
 483              **************************/
 484
 485             /* COULOMB ELECTROSTATICS */
 486             velec            = _mm_mul_pd(qq21,rinv21);
 487             felec            = _mm_mul_pd(velec,rinvsq21);
 488
 489             /* Update potential sum for this i atom from the interaction with this j atom. */
 490             velecsum         = _mm_add_pd(velecsum,velec);
 491
 492             fscal            = felec;
 493
 494             /* Update vectorial force */
 495             fix2             = _mm_macc_pd(dx21,fscal,fix2);
 496             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
 497             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
 498
 499             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
 500             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
 501             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
 502
 503             /**************************
 504              * CALCULATE INTERACTIONS *
 505              **************************/
 506
 507             /* COULOMB ELECTROSTATICS */
 508             velec            = _mm_mul_pd(qq22,rinv22);
 509             felec            = _mm_mul_pd(velec,rinvsq22);
 510
 511             /* Update potential sum for this i atom from the interaction with this j atom. */
 512             velecsum         = _mm_add_pd(velecsum,velec);
 513
 514             fscal            = felec;
 515
 516             /* Update vectorial force */
 517             fix2             = _mm_macc_pd(dx22,fscal,fix2);
 518             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
 519             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
 520
 521             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
 522             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
 523             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
 524
 525             gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 526
 527             /* Inner loop uses 314 flops */
 528         }
 529
 530         if(jidx<j_index_end)
 531         {
 532
 533             jnrA             = jjnr[jidx];
 534             j_coord_offsetA  = DIM*jnrA;
 535
 536             /* load j atom coordinates */
 537             gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
 538                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 539
 540             /* Calculate displacement vector */
 541             dx00             = _mm_sub_pd(ix0,jx0);
 542             dy00             = _mm_sub_pd(iy0,jy0);
 543             dz00             = _mm_sub_pd(iz0,jz0);
 544             dx01             = _mm_sub_pd(ix0,jx1);
 545             dy01             = _mm_sub_pd(iy0,jy1);
 546             dz01             = _mm_sub_pd(iz0,jz1);
 547             dx02             = _mm_sub_pd(ix0,jx2);
 548             dy02             = _mm_sub_pd(iy0,jy2);
 549             dz02             = _mm_sub_pd(iz0,jz2);
 550             dx10             = _mm_sub_pd(ix1,jx0);
 551             dy10             = _mm_sub_pd(iy1,jy0);
 552             dz10             = _mm_sub_pd(iz1,jz0);
 553             dx11             = _mm_sub_pd(ix1,jx1);
 554             dy11             = _mm_sub_pd(iy1,jy1);
 555             dz11             = _mm_sub_pd(iz1,jz1);
 556             dx12             = _mm_sub_pd(ix1,jx2);
 557             dy12             = _mm_sub_pd(iy1,jy2);
 558             dz12             = _mm_sub_pd(iz1,jz2);
 559             dx20             = _mm_sub_pd(ix2,jx0);
 560             dy20             = _mm_sub_pd(iy2,jy0);
 561             dz20             = _mm_sub_pd(iz2,jz0);
 562             dx21             = _mm_sub_pd(ix2,jx1);
 563             dy21             = _mm_sub_pd(iy2,jy1);
 564             dz21             = _mm_sub_pd(iz2,jz1);
 565             dx22             = _mm_sub_pd(ix2,jx2);
 566             dy22             = _mm_sub_pd(iy2,jy2);
 567             dz22             = _mm_sub_pd(iz2,jz2);
 568
 569             /* Calculate squared distance and things based on it */
 570             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 571             rsq01            = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
 572             rsq02            = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
 573             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 574             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
 575             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
 576             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 577             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
 578             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
 579
 580             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 581             rinv01           = gmx_mm_invsqrt_pd(rsq01);
 582             rinv02           = gmx_mm_invsqrt_pd(rsq02);
 583             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 584             rinv11           = gmx_mm_invsqrt_pd(rsq11);
 585             rinv12           = gmx_mm_invsqrt_pd(rsq12);
 586             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 587             rinv21           = gmx_mm_invsqrt_pd(rsq21);
 588             rinv22           = gmx_mm_invsqrt_pd(rsq22);
 589
 590             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
 591             rinvsq01         = _mm_mul_pd(rinv01,rinv01);
 592             rinvsq02         = _mm_mul_pd(rinv02,rinv02);
 593             rinvsq10         = _mm_mul_pd(rinv10,rinv10);
 594             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
 595             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
 596             rinvsq20         = _mm_mul_pd(rinv20,rinv20);
 597             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
 598             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
 599
 600             fjx0             = _mm_setzero_pd();
 601             fjy0             = _mm_setzero_pd();
 602             fjz0             = _mm_setzero_pd();
 603             fjx1             = _mm_setzero_pd();
 604             fjy1             = _mm_setzero_pd();
 605             fjz1             = _mm_setzero_pd();
 606             fjx2             = _mm_setzero_pd();
 607             fjy2             = _mm_setzero_pd();
 608             fjz2             = _mm_setzero_pd();
 609
 610             /**************************
 611              * CALCULATE INTERACTIONS *
 612              **************************/
 613
 614             r00              = _mm_mul_pd(rsq00,rinv00);
 615
 616             /* Calculate table index by multiplying r with table scale and truncate to integer */
 617             rt               = _mm_mul_pd(r00,vftabscale);
 618             vfitab           = _mm_cvttpd_epi32(rt);
 619 #ifdef __XOP__
 620             vfeps            = _mm_frcz_pd(rt);
 621 #else
 622             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 623 #endif
 624             twovfeps         = _mm_add_pd(vfeps,vfeps);
 625             vfitab           = _mm_slli_epi32(vfitab,3);
 626
 627             /* COULOMB ELECTROSTATICS */
 628             velec            = _mm_mul_pd(qq00,rinv00);
 629             felec            = _mm_mul_pd(velec,rinvsq00);
 630
 631             /* CUBIC SPLINE TABLE DISPERSION */
 632             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
 633             F                = _mm_setzero_pd();
 634             GMX_MM_TRANSPOSE2_PD(Y,F);
 635             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
 636             H                = _mm_setzero_pd();
 637             GMX_MM_TRANSPOSE2_PD(G,H);
 638             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
 639             VV               = _mm_macc_pd(vfeps,Fp,Y);
 640             vvdw6            = _mm_mul_pd(c6_00,VV);
 641             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
 642             fvdw6            = _mm_mul_pd(c6_00,FF);
 643
 644             /* CUBIC SPLINE TABLE REPULSION */
 645             vfitab           = _mm_add_epi32(vfitab,ifour);
 646             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
 647             F                = _mm_setzero_pd();
 648             GMX_MM_TRANSPOSE2_PD(Y,F);
 649             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
 650             H                = _mm_setzero_pd();
 651             GMX_MM_TRANSPOSE2_PD(G,H);
 652             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
 653             VV               = _mm_macc_pd(vfeps,Fp,Y);
 654             vvdw12           = _mm_mul_pd(c12_00,VV);
 655             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
 656             fvdw12           = _mm_mul_pd(c12_00,FF);
 657             vvdw             = _mm_add_pd(vvdw12,vvdw6);
 658             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
 659
 660             /* Update potential sum for this i atom from the interaction with this j atom. */
 661             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 662             velecsum         = _mm_add_pd(velecsum,velec);
 663             vvdw             = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
 664             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 665
 666             fscal            = _mm_add_pd(felec,fvdw);
 667
 668             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 669
 670             /* Update vectorial force */
 671             fix0             = _mm_macc_pd(dx00,fscal,fix0);
 672             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
 673             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
 674
 675             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
 676             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
 677             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
 678
 679             /**************************
 680              * CALCULATE INTERACTIONS *
 681              **************************/
 682
 683             /* COULOMB ELECTROSTATICS */
 684             velec            = _mm_mul_pd(qq01,rinv01);
 685             felec            = _mm_mul_pd(velec,rinvsq01);
 686
 687             /* Update potential sum for this i atom from the interaction with this j atom. */
 688             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 689             velecsum         = _mm_add_pd(velecsum,velec);
 690
 691             fscal            = felec;
 692
 693             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 694
 695             /* Update vectorial force */
 696             fix0             = _mm_macc_pd(dx01,fscal,fix0);
 697             fiy0             = _mm_macc_pd(dy01,fscal,fiy0);
 698             fiz0             = _mm_macc_pd(dz01,fscal,fiz0);
 699
 700             fjx1             = _mm_macc_pd(dx01,fscal,fjx1);
 701             fjy1             = _mm_macc_pd(dy01,fscal,fjy1);
 702             fjz1             = _mm_macc_pd(dz01,fscal,fjz1);
 703
 704             /**************************
 705              * CALCULATE INTERACTIONS *
 706              **************************/
 707
 708             /* COULOMB ELECTROSTATICS */
 709             velec            = _mm_mul_pd(qq02,rinv02);
 710             felec            = _mm_mul_pd(velec,rinvsq02);
 711
 712             /* Update potential sum for this i atom from the interaction with this j atom. */
 713             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 714             velecsum         = _mm_add_pd(velecsum,velec);
 715
 716             fscal            = felec;
 717
 718             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 719
 720             /* Update vectorial force */
 721             fix0             = _mm_macc_pd(dx02,fscal,fix0);
 722             fiy0             = _mm_macc_pd(dy02,fscal,fiy0);
 723             fiz0             = _mm_macc_pd(dz02,fscal,fiz0);
 724
 725             fjx2             = _mm_macc_pd(dx02,fscal,fjx2);
 726             fjy2             = _mm_macc_pd(dy02,fscal,fjy2);
 727             fjz2             = _mm_macc_pd(dz02,fscal,fjz2);
 728
 729             /**************************
 730              * CALCULATE INTERACTIONS *
 731              **************************/
 732
 733             /* COULOMB ELECTROSTATICS */
 734             velec            = _mm_mul_pd(qq10,rinv10);
 735             felec            = _mm_mul_pd(velec,rinvsq10);
 736
 737             /* Update potential sum for this i atom from the interaction with this j atom. */
 738             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 739             velecsum         = _mm_add_pd(velecsum,velec);
 740
 741             fscal            = felec;
 742
 743             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 744
 745             /* Update vectorial force */
 746             fix1             = _mm_macc_pd(dx10,fscal,fix1);
 747             fiy1             = _mm_macc_pd(dy10,fscal,fiy1);
 748             fiz1             = _mm_macc_pd(dz10,fscal,fiz1);
 749
 750             fjx0             = _mm_macc_pd(dx10,fscal,fjx0);
 751             fjy0             = _mm_macc_pd(dy10,fscal,fjy0);
 752             fjz0             = _mm_macc_pd(dz10,fscal,fjz0);
 753
 754             /**************************
 755              * CALCULATE INTERACTIONS *
 756              **************************/
 757
 758             /* COULOMB ELECTROSTATICS */
 759             velec            = _mm_mul_pd(qq11,rinv11);
 760             felec            = _mm_mul_pd(velec,rinvsq11);
 761
 762             /* Update potential sum for this i atom from the interaction with this j atom. */
 763             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 764             velecsum         = _mm_add_pd(velecsum,velec);
 765
 766             fscal            = felec;
 767
 768             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 769
 770             /* Update vectorial force */
 771             fix1             = _mm_macc_pd(dx11,fscal,fix1);
 772             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
 773             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
 774
 775             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
 776             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
 777             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
 778
 779             /**************************
 780              * CALCULATE INTERACTIONS *
 781              **************************/
 782
 783             /* COULOMB ELECTROSTATICS */
 784             velec            = _mm_mul_pd(qq12,rinv12);
 785             felec            = _mm_mul_pd(velec,rinvsq12);
 786
 787             /* Update potential sum for this i atom from the interaction with this j atom. */
 788             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 789             velecsum         = _mm_add_pd(velecsum,velec);
 790
 791             fscal            = felec;
 792
 793             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 794
 795             /* Update vectorial force */
 796             fix1             = _mm_macc_pd(dx12,fscal,fix1);
 797             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
 798             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
 799
 800             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
 801             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
 802             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
 803
 804             /**************************
 805              * CALCULATE INTERACTIONS *
 806              **************************/
 807
 808             /* COULOMB ELECTROSTATICS */
 809             velec            = _mm_mul_pd(qq20,rinv20);
 810             felec            = _mm_mul_pd(velec,rinvsq20);
 811
 812             /* Update potential sum for this i atom from the interaction with this j atom. */
 813             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 814             velecsum         = _mm_add_pd(velecsum,velec);
 815
 816             fscal            = felec;
 817
 818             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 819
 820             /* Update vectorial force */
 821             fix2             = _mm_macc_pd(dx20,fscal,fix2);
 822             fiy2             = _mm_macc_pd(dy20,fscal,fiy2);
 823             fiz2             = _mm_macc_pd(dz20,fscal,fiz2);
 824
 825             fjx0             = _mm_macc_pd(dx20,fscal,fjx0);
 826             fjy0             = _mm_macc_pd(dy20,fscal,fjy0);
 827             fjz0             = _mm_macc_pd(dz20,fscal,fjz0);
 828
 829             /**************************
 830              * CALCULATE INTERACTIONS *
 831              **************************/
 832
 833             /* COULOMB ELECTROSTATICS */
 834             velec            = _mm_mul_pd(qq21,rinv21);
 835             felec            = _mm_mul_pd(velec,rinvsq21);
 836
 837             /* Update potential sum for this i atom from the interaction with this j atom. */
 838             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 839             velecsum         = _mm_add_pd(velecsum,velec);
 840
 841             fscal            = felec;
 842
 843             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 844
 845             /* Update vectorial force */
 846             fix2             = _mm_macc_pd(dx21,fscal,fix2);
 847             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
 848             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
 849
 850             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
 851             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
 852             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
 853
 854             /**************************
 855              * CALCULATE INTERACTIONS *
 856              **************************/
 857
 858             /* COULOMB ELECTROSTATICS */
 859             velec            = _mm_mul_pd(qq22,rinv22);
 860             felec            = _mm_mul_pd(velec,rinvsq22);
 861
 862             /* Update potential sum for this i atom from the interaction with this j atom. */
 863             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 864             velecsum         = _mm_add_pd(velecsum,velec);
 865
 866             fscal            = felec;
 867
 868             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 869
 870             /* Update vectorial force */
 871             fix2             = _mm_macc_pd(dx22,fscal,fix2);
 872             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
 873             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
 874
 875             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
 876             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
 877             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
 878
 879             gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 880
 881             /* Inner loop uses 314 flops */
 882         }
 883
 884         /* End of innermost loop */
 885
 886         gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
 887                                               f+i_coord_offset,fshift+i_shift_offset);
 888
 889         ggid                        = gid[iidx];
 890         /* Update potential energies */
 891         gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
 892         gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
 893
 894         /* Increment number of inner iterations */
 895         inneriter                  += j_index_end - j_index_start;
 896
 897         /* Outer loop uses 20 flops */
 898     }
 899
 900     /* Increment number of outer iterations */
 901     outeriter        += nri;
 902
 903     /* Update outer/inner flops */
 904
 905     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*314);
 906 }
 907 /*
 908  * Gromacs nonbonded kernel:   nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_F_avx_128_fma_double
 909  * Electrostatics interaction: Coulomb
 910  * VdW interaction:            CubicSplineTable
 911  * Geometry:                   Water3-Water3
 912  * Calculate force/pot:        Force
 913  */
 914 void
 915 nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_F_avx_128_fma_double
 916                     (t_nblist                    * gmx_restrict       nlist,
 917                      rvec                        * gmx_restrict          xx,
 918                      rvec                        * gmx_restrict          ff,
 919                      t_forcerec                  * gmx_restrict          fr,
 920                      t_mdatoms                   * gmx_restrict     mdatoms,
 921                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
 922                      t_nrnb                      * gmx_restrict        nrnb)
 923 {
 924     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 925      * just 0 for non-waters.
 926      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
 927      * jnr indices corresponding to data put in the four positions in the SIMD register.
 928      */
 929     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 930     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 931     int              jnrA,jnrB;
 932     int              j_coord_offsetA,j_coord_offsetB;
 933     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 934     real             rcutoff_scalar;
 935     real             *shiftvec,*fshift,*x,*f;
 936     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
 937     int              vdwioffset0;
 938     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 939     int              vdwioffset1;
 940     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 941     int              vdwioffset2;
 942     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 943     int              vdwjidx0A,vdwjidx0B;
 944     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 945     int              vdwjidx1A,vdwjidx1B;
 946     __m128d          jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
 947     int              vdwjidx2A,vdwjidx2B;
 948     __m128d          jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
 949     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 950     __m128d          dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
 951     __m128d          dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
 952     __m128d          dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 953     __m128d          dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 954     __m128d          dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 955     __m128d          dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 956     __m128d          dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 957     __m128d          dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 958     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
 959     real             *charge;
 960     int              nvdwtype;
 961     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 962     int              *vdwtype;
 963     real             *vdwparam;
 964     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
 965     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
 966     __m128i          vfitab;
 967     __m128i          ifour       = _mm_set1_epi32(4);
 968     __m128d          rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF,twovfeps;
 969     real             *vftab;
 970     __m128d          dummy_mask,cutoff_mask;
 971     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
 972     __m128d          one     = _mm_set1_pd(1.0);
 973     __m128d          two     = _mm_set1_pd(2.0);
 974     x                = xx[0];
 975     f                = ff[0];
 976
 977     nri              = nlist->nri;
 978     iinr             = nlist->iinr;
 979     jindex           = nlist->jindex;
 980     jjnr             = nlist->jjnr;
 981     shiftidx         = nlist->shift;
 982     gid              = nlist->gid;
 983     shiftvec         = fr->shift_vec[0];
 984     fshift           = fr->fshift[0];
 985     facel            = _mm_set1_pd(fr->epsfac);
 986     charge           = mdatoms->chargeA;
 987     nvdwtype         = fr->ntype;
 988     vdwparam         = fr->nbfp;
 989     vdwtype          = mdatoms->typeA;
 990
 991     vftab            = kernel_data->table_vdw->data;
 992     vftabscale       = _mm_set1_pd(kernel_data->table_vdw->scale);
 993
 994     /* Setup water-specific parameters */
 995     inr              = nlist->iinr[0];
 996     iq0              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
 997     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
 998     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
 999     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
1000
1001     jq0              = _mm_set1_pd(charge[inr+0]);
1002     jq1              = _mm_set1_pd(charge[inr+1]);
1003     jq2              = _mm_set1_pd(charge[inr+2]);
1004     vdwjidx0A        = 2*vdwtype[inr+0];
1005     qq00             = _mm_mul_pd(iq0,jq0);
1006     c6_00            = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
1007     c12_00           = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
1008     qq01             = _mm_mul_pd(iq0,jq1);
1009     qq02             = _mm_mul_pd(iq0,jq2);
1010     qq10             = _mm_mul_pd(iq1,jq0);
1011     qq11             = _mm_mul_pd(iq1,jq1);
1012     qq12             = _mm_mul_pd(iq1,jq2);
1013     qq20             = _mm_mul_pd(iq2,jq0);
1014     qq21             = _mm_mul_pd(iq2,jq1);
1015     qq22             = _mm_mul_pd(iq2,jq2);
1016
1017     /* Avoid stupid compiler warnings */
1018     jnrA = jnrB = 0;
1019     j_coord_offsetA = 0;
1020     j_coord_offsetB = 0;
1021
1022     outeriter        = 0;
1023     inneriter        = 0;
1024
1025     /* Start outer loop over neighborlists */
1026     for(iidx=0; iidx<nri; iidx++)
1027     {
1028         /* Load shift vector for this list */
1029         i_shift_offset   = DIM*shiftidx[iidx];
1030
1031         /* Load limits for loop over neighbors */
1032         j_index_start    = jindex[iidx];
1033         j_index_end      = jindex[iidx+1];
1034
1035         /* Get outer coordinate index */
1036         inr              = iinr[iidx];
1037         i_coord_offset   = DIM*inr;
1038
1039         /* Load i particle coords and add shift vector */
1040         gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
1041                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1042
1043         fix0             = _mm_setzero_pd();
1044         fiy0             = _mm_setzero_pd();
1045         fiz0             = _mm_setzero_pd();
1046         fix1             = _mm_setzero_pd();
1047         fiy1             = _mm_setzero_pd();
1048         fiz1             = _mm_setzero_pd();
1049         fix2             = _mm_setzero_pd();
1050         fiy2             = _mm_setzero_pd();
1051         fiz2             = _mm_setzero_pd();
1052
1053         /* Start inner kernel loop */
1054         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
1055         {
1056
1057             /* Get j neighbor index, and coordinate index */
1058             jnrA             = jjnr[jidx];
1059             jnrB             = jjnr[jidx+1];
1060             j_coord_offsetA  = DIM*jnrA;
1061             j_coord_offsetB  = DIM*jnrB;
1062
1063             /* load j atom coordinates */
1064             gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1065                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1066
1067             /* Calculate displacement vector */
1068             dx00             = _mm_sub_pd(ix0,jx0);
1069             dy00             = _mm_sub_pd(iy0,jy0);
1070             dz00             = _mm_sub_pd(iz0,jz0);
1071             dx01             = _mm_sub_pd(ix0,jx1);
1072             dy01             = _mm_sub_pd(iy0,jy1);
1073             dz01             = _mm_sub_pd(iz0,jz1);
1074             dx02             = _mm_sub_pd(ix0,jx2);
1075             dy02             = _mm_sub_pd(iy0,jy2);
1076             dz02             = _mm_sub_pd(iz0,jz2);
1077             dx10             = _mm_sub_pd(ix1,jx0);
1078             dy10             = _mm_sub_pd(iy1,jy0);
1079             dz10             = _mm_sub_pd(iz1,jz0);
1080             dx11             = _mm_sub_pd(ix1,jx1);
1081             dy11             = _mm_sub_pd(iy1,jy1);
1082             dz11             = _mm_sub_pd(iz1,jz1);
1083             dx12             = _mm_sub_pd(ix1,jx2);
1084             dy12             = _mm_sub_pd(iy1,jy2);
1085             dz12             = _mm_sub_pd(iz1,jz2);
1086             dx20             = _mm_sub_pd(ix2,jx0);
1087             dy20             = _mm_sub_pd(iy2,jy0);
1088             dz20             = _mm_sub_pd(iz2,jz0);
1089             dx21             = _mm_sub_pd(ix2,jx1);
1090             dy21             = _mm_sub_pd(iy2,jy1);
1091             dz21             = _mm_sub_pd(iz2,jz1);
1092             dx22             = _mm_sub_pd(ix2,jx2);
1093             dy22             = _mm_sub_pd(iy2,jy2);
1094             dz22             = _mm_sub_pd(iz2,jz2);
1095
1096             /* Calculate squared distance and things based on it */
1097             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1098             rsq01            = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1099             rsq02            = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1100             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1101             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1102             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1103             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1104             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1105             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1106
1107             rinv00           = gmx_mm_invsqrt_pd(rsq00);
1108             rinv01           = gmx_mm_invsqrt_pd(rsq01);
1109             rinv02           = gmx_mm_invsqrt_pd(rsq02);
1110             rinv10           = gmx_mm_invsqrt_pd(rsq10);
1111             rinv11           = gmx_mm_invsqrt_pd(rsq11);
1112             rinv12           = gmx_mm_invsqrt_pd(rsq12);
1113             rinv20           = gmx_mm_invsqrt_pd(rsq20);
1114             rinv21           = gmx_mm_invsqrt_pd(rsq21);
1115             rinv22           = gmx_mm_invsqrt_pd(rsq22);
1116
1117             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
1118             rinvsq01         = _mm_mul_pd(rinv01,rinv01);
1119             rinvsq02         = _mm_mul_pd(rinv02,rinv02);
1120             rinvsq10         = _mm_mul_pd(rinv10,rinv10);
1121             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
1122             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
1123             rinvsq20         = _mm_mul_pd(rinv20,rinv20);
1124             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
1125             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
1126
1127             fjx0             = _mm_setzero_pd();
1128             fjy0             = _mm_setzero_pd();
1129             fjz0             = _mm_setzero_pd();
1130             fjx1             = _mm_setzero_pd();
1131             fjy1             = _mm_setzero_pd();
1132             fjz1             = _mm_setzero_pd();
1133             fjx2             = _mm_setzero_pd();
1134             fjy2             = _mm_setzero_pd();
1135             fjz2             = _mm_setzero_pd();
1136
1137             /**************************
1138              * CALCULATE INTERACTIONS *
1139              **************************/
1140
1141             r00              = _mm_mul_pd(rsq00,rinv00);
1142
1143             /* Calculate table index by multiplying r with table scale and truncate to integer */
1144             rt               = _mm_mul_pd(r00,vftabscale);
1145             vfitab           = _mm_cvttpd_epi32(rt);
1146 #ifdef __XOP__
1147             vfeps            = _mm_frcz_pd(rt);
1148 #else
1149             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1150 #endif
1151             twovfeps         = _mm_add_pd(vfeps,vfeps);
1152             vfitab           = _mm_slli_epi32(vfitab,3);
1153
1154             /* COULOMB ELECTROSTATICS */
1155             velec            = _mm_mul_pd(qq00,rinv00);
1156             felec            = _mm_mul_pd(velec,rinvsq00);
1157
1158             /* CUBIC SPLINE TABLE DISPERSION */
1159             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1160             F                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1161             GMX_MM_TRANSPOSE2_PD(Y,F);
1162             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1163             H                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
1164             GMX_MM_TRANSPOSE2_PD(G,H);
1165             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1166             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1167             fvdw6            = _mm_mul_pd(c6_00,FF);
1168
1169             /* CUBIC SPLINE TABLE REPULSION */
1170             vfitab           = _mm_add_epi32(vfitab,ifour);
1171             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1172             F                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1173             GMX_MM_TRANSPOSE2_PD(Y,F);
1174             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1175             H                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
1176             GMX_MM_TRANSPOSE2_PD(G,H);
1177             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1178             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1179             fvdw12           = _mm_mul_pd(c12_00,FF);
1180             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
1181
1182             fscal            = _mm_add_pd(felec,fvdw);
1183
1184             /* Update vectorial force */
1185             fix0             = _mm_macc_pd(dx00,fscal,fix0);
1186             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
1187             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
1188
1189             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
1190             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
1191             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
1192
1193             /**************************
1194              * CALCULATE INTERACTIONS *
1195              **************************/
1196
1197             /* COULOMB ELECTROSTATICS */
1198             velec            = _mm_mul_pd(qq01,rinv01);
1199             felec            = _mm_mul_pd(velec,rinvsq01);
1200
1201             fscal            = felec;
1202
1203             /* Update vectorial force */
1204             fix0             = _mm_macc_pd(dx01,fscal,fix0);
1205             fiy0             = _mm_macc_pd(dy01,fscal,fiy0);
1206             fiz0             = _mm_macc_pd(dz01,fscal,fiz0);
1207
1208             fjx1             = _mm_macc_pd(dx01,fscal,fjx1);
1209             fjy1             = _mm_macc_pd(dy01,fscal,fjy1);
1210             fjz1             = _mm_macc_pd(dz01,fscal,fjz1);
1211
1212             /**************************
1213              * CALCULATE INTERACTIONS *
1214              **************************/
1215
1216             /* COULOMB ELECTROSTATICS */
1217             velec            = _mm_mul_pd(qq02,rinv02);
1218             felec            = _mm_mul_pd(velec,rinvsq02);
1219
1220             fscal            = felec;
1221
1222             /* Update vectorial force */
1223             fix0             = _mm_macc_pd(dx02,fscal,fix0);
1224             fiy0             = _mm_macc_pd(dy02,fscal,fiy0);
1225             fiz0             = _mm_macc_pd(dz02,fscal,fiz0);
1226
1227             fjx2             = _mm_macc_pd(dx02,fscal,fjx2);
1228             fjy2             = _mm_macc_pd(dy02,fscal,fjy2);
1229             fjz2             = _mm_macc_pd(dz02,fscal,fjz2);
1230
1231             /**************************
1232              * CALCULATE INTERACTIONS *
1233              **************************/
1234
1235             /* COULOMB ELECTROSTATICS */
1236             velec            = _mm_mul_pd(qq10,rinv10);
1237             felec            = _mm_mul_pd(velec,rinvsq10);
1238
1239             fscal            = felec;
1240
1241             /* Update vectorial force */
1242             fix1             = _mm_macc_pd(dx10,fscal,fix1);
1243             fiy1             = _mm_macc_pd(dy10,fscal,fiy1);
1244             fiz1             = _mm_macc_pd(dz10,fscal,fiz1);
1245
1246             fjx0             = _mm_macc_pd(dx10,fscal,fjx0);
1247             fjy0             = _mm_macc_pd(dy10,fscal,fjy0);
1248             fjz0             = _mm_macc_pd(dz10,fscal,fjz0);
1249
1250             /**************************
1251              * CALCULATE INTERACTIONS *
1252              **************************/
1253
1254             /* COULOMB ELECTROSTATICS */
1255             velec            = _mm_mul_pd(qq11,rinv11);
1256             felec            = _mm_mul_pd(velec,rinvsq11);
1257
1258             fscal            = felec;
1259
1260             /* Update vectorial force */
1261             fix1             = _mm_macc_pd(dx11,fscal,fix1);
1262             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
1263             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
1264
1265             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
1266             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
1267             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
1268
1269             /**************************
1270              * CALCULATE INTERACTIONS *
1271              **************************/
1272
1273             /* COULOMB ELECTROSTATICS */
1274             velec            = _mm_mul_pd(qq12,rinv12);
1275             felec            = _mm_mul_pd(velec,rinvsq12);
1276
1277             fscal            = felec;
1278
1279             /* Update vectorial force */
1280             fix1             = _mm_macc_pd(dx12,fscal,fix1);
1281             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
1282             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
1283
1284             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
1285             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
1286             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
1287
1288             /**************************
1289              * CALCULATE INTERACTIONS *
1290              **************************/
1291
1292             /* COULOMB ELECTROSTATICS */
1293             velec            = _mm_mul_pd(qq20,rinv20);
1294             felec            = _mm_mul_pd(velec,rinvsq20);
1295
1296             fscal            = felec;
1297
1298             /* Update vectorial force */
1299             fix2             = _mm_macc_pd(dx20,fscal,fix2);
1300             fiy2             = _mm_macc_pd(dy20,fscal,fiy2);
1301             fiz2             = _mm_macc_pd(dz20,fscal,fiz2);
1302
1303             fjx0             = _mm_macc_pd(dx20,fscal,fjx0);
1304             fjy0             = _mm_macc_pd(dy20,fscal,fjy0);
1305             fjz0             = _mm_macc_pd(dz20,fscal,fjz0);
1306
1307             /**************************
1308              * CALCULATE INTERACTIONS *
1309              **************************/
1310
1311             /* COULOMB ELECTROSTATICS */
1312             velec            = _mm_mul_pd(qq21,rinv21);
1313             felec            = _mm_mul_pd(velec,rinvsq21);
1314
1315             fscal            = felec;
1316
1317             /* Update vectorial force */
1318             fix2             = _mm_macc_pd(dx21,fscal,fix2);
1319             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
1320             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
1321
1322             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
1323             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
1324             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
1325
1326             /**************************
1327              * CALCULATE INTERACTIONS *
1328              **************************/
1329
1330             /* COULOMB ELECTROSTATICS */
1331             velec            = _mm_mul_pd(qq22,rinv22);
1332             felec            = _mm_mul_pd(velec,rinvsq22);
1333
1334             fscal            = felec;
1335
1336             /* Update vectorial force */
1337             fix2             = _mm_macc_pd(dx22,fscal,fix2);
1338             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
1339             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
1340
1341             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
1342             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
1343             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
1344
1345             gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1346
1347             /* Inner loop uses 297 flops */
1348         }
1349
1350         if(jidx<j_index_end)
1351         {
1352
1353             jnrA             = jjnr[jidx];
1354             j_coord_offsetA  = DIM*jnrA;
1355
1356             /* load j atom coordinates */
1357             gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
1358                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1359
1360             /* Calculate displacement vector */
1361             dx00             = _mm_sub_pd(ix0,jx0);
1362             dy00             = _mm_sub_pd(iy0,jy0);
1363             dz00             = _mm_sub_pd(iz0,jz0);
1364             dx01             = _mm_sub_pd(ix0,jx1);
1365             dy01             = _mm_sub_pd(iy0,jy1);
1366             dz01             = _mm_sub_pd(iz0,jz1);
1367             dx02             = _mm_sub_pd(ix0,jx2);
1368             dy02             = _mm_sub_pd(iy0,jy2);
1369             dz02             = _mm_sub_pd(iz0,jz2);
1370             dx10             = _mm_sub_pd(ix1,jx0);
1371             dy10             = _mm_sub_pd(iy1,jy0);
1372             dz10             = _mm_sub_pd(iz1,jz0);
1373             dx11             = _mm_sub_pd(ix1,jx1);
1374             dy11             = _mm_sub_pd(iy1,jy1);
1375             dz11             = _mm_sub_pd(iz1,jz1);
1376             dx12             = _mm_sub_pd(ix1,jx2);
1377             dy12             = _mm_sub_pd(iy1,jy2);
1378             dz12             = _mm_sub_pd(iz1,jz2);
1379             dx20             = _mm_sub_pd(ix2,jx0);
1380             dy20             = _mm_sub_pd(iy2,jy0);
1381             dz20             = _mm_sub_pd(iz2,jz0);
1382             dx21             = _mm_sub_pd(ix2,jx1);
1383             dy21             = _mm_sub_pd(iy2,jy1);
1384             dz21             = _mm_sub_pd(iz2,jz1);
1385             dx22             = _mm_sub_pd(ix2,jx2);
1386             dy22             = _mm_sub_pd(iy2,jy2);
1387             dz22             = _mm_sub_pd(iz2,jz2);
1388
1389             /* Calculate squared distance and things based on it */
1390             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1391             rsq01            = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1392             rsq02            = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1393             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1394             rsq11            = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1395             rsq12            = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1396             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1397             rsq21            = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1398             rsq22            = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1399
1400             rinv00           = gmx_mm_invsqrt_pd(rsq00);
1401             rinv01           = gmx_mm_invsqrt_pd(rsq01);
1402             rinv02           = gmx_mm_invsqrt_pd(rsq02);
1403             rinv10           = gmx_mm_invsqrt_pd(rsq10);
1404             rinv11           = gmx_mm_invsqrt_pd(rsq11);
1405             rinv12           = gmx_mm_invsqrt_pd(rsq12);
1406             rinv20           = gmx_mm_invsqrt_pd(rsq20);
1407             rinv21           = gmx_mm_invsqrt_pd(rsq21);
1408             rinv22           = gmx_mm_invsqrt_pd(rsq22);
1409
1410             rinvsq00         = _mm_mul_pd(rinv00,rinv00);
1411             rinvsq01         = _mm_mul_pd(rinv01,rinv01);
1412             rinvsq02         = _mm_mul_pd(rinv02,rinv02);
1413             rinvsq10         = _mm_mul_pd(rinv10,rinv10);
1414             rinvsq11         = _mm_mul_pd(rinv11,rinv11);
1415             rinvsq12         = _mm_mul_pd(rinv12,rinv12);
1416             rinvsq20         = _mm_mul_pd(rinv20,rinv20);
1417             rinvsq21         = _mm_mul_pd(rinv21,rinv21);
1418             rinvsq22         = _mm_mul_pd(rinv22,rinv22);
1419
1420             fjx0             = _mm_setzero_pd();
1421             fjy0             = _mm_setzero_pd();
1422             fjz0             = _mm_setzero_pd();
1423             fjx1             = _mm_setzero_pd();
1424             fjy1             = _mm_setzero_pd();
1425             fjz1             = _mm_setzero_pd();
1426             fjx2             = _mm_setzero_pd();
1427             fjy2             = _mm_setzero_pd();
1428             fjz2             = _mm_setzero_pd();
1429
1430             /**************************
1431              * CALCULATE INTERACTIONS *
1432              **************************/
1433
1434             r00              = _mm_mul_pd(rsq00,rinv00);
1435
1436             /* Calculate table index by multiplying r with table scale and truncate to integer */
1437             rt               = _mm_mul_pd(r00,vftabscale);
1438             vfitab           = _mm_cvttpd_epi32(rt);
1439 #ifdef __XOP__
1440             vfeps            = _mm_frcz_pd(rt);
1441 #else
1442             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1443 #endif
1444             twovfeps         = _mm_add_pd(vfeps,vfeps);
1445             vfitab           = _mm_slli_epi32(vfitab,3);
1446
1447             /* COULOMB ELECTROSTATICS */
1448             velec            = _mm_mul_pd(qq00,rinv00);
1449             felec            = _mm_mul_pd(velec,rinvsq00);
1450
1451             /* CUBIC SPLINE TABLE DISPERSION */
1452             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1453             F                = _mm_setzero_pd();
1454             GMX_MM_TRANSPOSE2_PD(Y,F);
1455             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1456             H                = _mm_setzero_pd();
1457             GMX_MM_TRANSPOSE2_PD(G,H);
1458             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1459             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1460             fvdw6            = _mm_mul_pd(c6_00,FF);
1461
1462             /* CUBIC SPLINE TABLE REPULSION */
1463             vfitab           = _mm_add_epi32(vfitab,ifour);
1464             Y                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1465             F                = _mm_setzero_pd();
1466             GMX_MM_TRANSPOSE2_PD(Y,F);
1467             G                = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1468             H                = _mm_setzero_pd();
1469             GMX_MM_TRANSPOSE2_PD(G,H);
1470             Fp               = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1471             FF               = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1472             fvdw12           = _mm_mul_pd(c12_00,FF);
1473             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
1474
1475             fscal            = _mm_add_pd(felec,fvdw);
1476
1477             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1478
1479             /* Update vectorial force */
1480             fix0             = _mm_macc_pd(dx00,fscal,fix0);
1481             fiy0             = _mm_macc_pd(dy00,fscal,fiy0);
1482             fiz0             = _mm_macc_pd(dz00,fscal,fiz0);
1483
1484             fjx0             = _mm_macc_pd(dx00,fscal,fjx0);
1485             fjy0             = _mm_macc_pd(dy00,fscal,fjy0);
1486             fjz0             = _mm_macc_pd(dz00,fscal,fjz0);
1487
1488             /**************************
1489              * CALCULATE INTERACTIONS *
1490              **************************/
1491
1492             /* COULOMB ELECTROSTATICS */
1493             velec            = _mm_mul_pd(qq01,rinv01);
1494             felec            = _mm_mul_pd(velec,rinvsq01);
1495
1496             fscal            = felec;
1497
1498             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1499
1500             /* Update vectorial force */
1501             fix0             = _mm_macc_pd(dx01,fscal,fix0);
1502             fiy0             = _mm_macc_pd(dy01,fscal,fiy0);
1503             fiz0             = _mm_macc_pd(dz01,fscal,fiz0);
1504
1505             fjx1             = _mm_macc_pd(dx01,fscal,fjx1);
1506             fjy1             = _mm_macc_pd(dy01,fscal,fjy1);
1507             fjz1             = _mm_macc_pd(dz01,fscal,fjz1);
1508
1509             /**************************
1510              * CALCULATE INTERACTIONS *
1511              **************************/
1512
1513             /* COULOMB ELECTROSTATICS */
1514             velec            = _mm_mul_pd(qq02,rinv02);
1515             felec            = _mm_mul_pd(velec,rinvsq02);
1516
1517             fscal            = felec;
1518
1519             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1520
1521             /* Update vectorial force */
1522             fix0             = _mm_macc_pd(dx02,fscal,fix0);
1523             fiy0             = _mm_macc_pd(dy02,fscal,fiy0);
1524             fiz0             = _mm_macc_pd(dz02,fscal,fiz0);
1525
1526             fjx2             = _mm_macc_pd(dx02,fscal,fjx2);
1527             fjy2             = _mm_macc_pd(dy02,fscal,fjy2);
1528             fjz2             = _mm_macc_pd(dz02,fscal,fjz2);
1529
1530             /**************************
1531              * CALCULATE INTERACTIONS *
1532              **************************/
1533
1534             /* COULOMB ELECTROSTATICS */
1535             velec            = _mm_mul_pd(qq10,rinv10);
1536             felec            = _mm_mul_pd(velec,rinvsq10);
1537
1538             fscal            = felec;
1539
1540             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1541
1542             /* Update vectorial force */
1543             fix1             = _mm_macc_pd(dx10,fscal,fix1);
1544             fiy1             = _mm_macc_pd(dy10,fscal,fiy1);
1545             fiz1             = _mm_macc_pd(dz10,fscal,fiz1);
1546
1547             fjx0             = _mm_macc_pd(dx10,fscal,fjx0);
1548             fjy0             = _mm_macc_pd(dy10,fscal,fjy0);
1549             fjz0             = _mm_macc_pd(dz10,fscal,fjz0);
1550
1551             /**************************
1552              * CALCULATE INTERACTIONS *
1553              **************************/
1554
1555             /* COULOMB ELECTROSTATICS */
1556             velec            = _mm_mul_pd(qq11,rinv11);
1557             felec            = _mm_mul_pd(velec,rinvsq11);
1558
1559             fscal            = felec;
1560
1561             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1562
1563             /* Update vectorial force */
1564             fix1             = _mm_macc_pd(dx11,fscal,fix1);
1565             fiy1             = _mm_macc_pd(dy11,fscal,fiy1);
1566             fiz1             = _mm_macc_pd(dz11,fscal,fiz1);
1567
1568             fjx1             = _mm_macc_pd(dx11,fscal,fjx1);
1569             fjy1             = _mm_macc_pd(dy11,fscal,fjy1);
1570             fjz1             = _mm_macc_pd(dz11,fscal,fjz1);
1571
1572             /**************************
1573              * CALCULATE INTERACTIONS *
1574              **************************/
1575
1576             /* COULOMB ELECTROSTATICS */
1577             velec            = _mm_mul_pd(qq12,rinv12);
1578             felec            = _mm_mul_pd(velec,rinvsq12);
1579
1580             fscal            = felec;
1581
1582             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1583
1584             /* Update vectorial force */
1585             fix1             = _mm_macc_pd(dx12,fscal,fix1);
1586             fiy1             = _mm_macc_pd(dy12,fscal,fiy1);
1587             fiz1             = _mm_macc_pd(dz12,fscal,fiz1);
1588
1589             fjx2             = _mm_macc_pd(dx12,fscal,fjx2);
1590             fjy2             = _mm_macc_pd(dy12,fscal,fjy2);
1591             fjz2             = _mm_macc_pd(dz12,fscal,fjz2);
1592
1593             /**************************
1594              * CALCULATE INTERACTIONS *
1595              **************************/
1596
1597             /* COULOMB ELECTROSTATICS */
1598             velec            = _mm_mul_pd(qq20,rinv20);
1599             felec            = _mm_mul_pd(velec,rinvsq20);
1600
1601             fscal            = felec;
1602
1603             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1604
1605             /* Update vectorial force */
1606             fix2             = _mm_macc_pd(dx20,fscal,fix2);
1607             fiy2             = _mm_macc_pd(dy20,fscal,fiy2);
1608             fiz2             = _mm_macc_pd(dz20,fscal,fiz2);
1609
1610             fjx0             = _mm_macc_pd(dx20,fscal,fjx0);
1611             fjy0             = _mm_macc_pd(dy20,fscal,fjy0);
1612             fjz0             = _mm_macc_pd(dz20,fscal,fjz0);
1613
1614             /**************************
1615              * CALCULATE INTERACTIONS *
1616              **************************/
1617
1618             /* COULOMB ELECTROSTATICS */
1619             velec            = _mm_mul_pd(qq21,rinv21);
1620             felec            = _mm_mul_pd(velec,rinvsq21);
1621
1622             fscal            = felec;
1623
1624             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1625
1626             /* Update vectorial force */
1627             fix2             = _mm_macc_pd(dx21,fscal,fix2);
1628             fiy2             = _mm_macc_pd(dy21,fscal,fiy2);
1629             fiz2             = _mm_macc_pd(dz21,fscal,fiz2);
1630
1631             fjx1             = _mm_macc_pd(dx21,fscal,fjx1);
1632             fjy1             = _mm_macc_pd(dy21,fscal,fjy1);
1633             fjz1             = _mm_macc_pd(dz21,fscal,fjz1);
1634
1635             /**************************
1636              * CALCULATE INTERACTIONS *
1637              **************************/
1638
1639             /* COULOMB ELECTROSTATICS */
1640             velec            = _mm_mul_pd(qq22,rinv22);
1641             felec            = _mm_mul_pd(velec,rinvsq22);
1642
1643             fscal            = felec;
1644
1645             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1646
1647             /* Update vectorial force */
1648             fix2             = _mm_macc_pd(dx22,fscal,fix2);
1649             fiy2             = _mm_macc_pd(dy22,fscal,fiy2);
1650             fiz2             = _mm_macc_pd(dz22,fscal,fiz2);
1651
1652             fjx2             = _mm_macc_pd(dx22,fscal,fjx2);
1653             fjy2             = _mm_macc_pd(dy22,fscal,fjy2);
1654             fjz2             = _mm_macc_pd(dz22,fscal,fjz2);
1655
1656             gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1657
1658             /* Inner loop uses 297 flops */
1659         }
1660
1661         /* End of innermost loop */
1662
1663         gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1664                                               f+i_coord_offset,fshift+i_shift_offset);
1665
1666         /* Increment number of inner iterations */
1667         inneriter                  += j_index_end - j_index_start;
1668
1669         /* Outer loop uses 18 flops */
1670     }
1671
1672     /* Increment number of outer iterations */
1673     outeriter        += nri;
1674
1675     /* Update outer/inner flops */
1676
1677     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*297);
1678 }