src/gmxlib/nonbonded/nb_kernel_sse4_1_double/nb_kernel_ElecCSTab_VdwCSTab_GeomW3P1_sse4_1_double.c

   1 /*
   2  * Note: this file was generated by the Gromacs sse4_1_double kernel generator.
   3  *
   4  *                This source code is part of
   5  *
   6  *                 G   R   O   M   A   C   S
   7  *
   8  * Copyright (c) 2001-2012, The GROMACS Development Team
   9  *
  10  * Gromacs is a library for molecular simulation and trajectory analysis,
  11  * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
  12  * a full list of developers and information, check out http://www.gromacs.org
  13  *
  14  * This program is free software; you can redistribute it and/or modify it under
  15  * the terms of the GNU Lesser General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option) any
  17  * later version.
  18  *
  19  * To help fund GROMACS development, we humbly ask that you cite
  20  * the papers people have written on it - you can find them on the website.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include <config.h>
  24 #endif
  25
  26 #include <math.h>
  27
  28 #include "../nb_kernel.h"
  29 #include "types/simple.h"
  30 #include "vec.h"
  31 #include "nrnb.h"
  32
  33 #include "gmx_math_x86_sse4_1_double.h"
  34 #include "kernelutil_x86_sse4_1_double.h"
  35
  36 /*
  37  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwCSTab_GeomW3P1_VF_sse4_1_double
  38  * Electrostatics interaction: CubicSplineTable
  39  * VdW interaction:            CubicSplineTable
  40  * Geometry:                   Water3-Particle
  41  * Calculate force/pot:        PotentialAndForce
  42  */
  43 void
  44 nb_kernel_ElecCSTab_VdwCSTab_GeomW3P1_VF_sse4_1_double
  45                     (t_nblist * gmx_restrict                nlist,
  46                      rvec * gmx_restrict                    xx,
  47                      rvec * gmx_restrict                    ff,
  48                      t_forcerec * gmx_restrict              fr,
  49                      t_mdatoms * gmx_restrict               mdatoms,
  50                      nb_kernel_data_t * gmx_restrict        kernel_data,
  51                      t_nrnb * gmx_restrict                  nrnb)
  52 {
  53     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  54      * just 0 for non-waters.
  55      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
  56      * jnr indices corresponding to data put in the four positions in the SIMD register.
  57      */
  58     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  59     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  60     int              jnrA,jnrB;
  61     int              j_coord_offsetA,j_coord_offsetB;
  62     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  63     real             rcutoff_scalar;
  64     real             *shiftvec,*fshift,*x,*f;
  65     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
  66     int              vdwioffset0;
  67     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  68     int              vdwioffset1;
  69     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  70     int              vdwioffset2;
  71     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  72     int              vdwjidx0A,vdwjidx0B;
  73     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  74     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  75     __m128d          dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  76     __m128d          dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
  77     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
  78     real             *charge;
  79     int              nvdwtype;
  80     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
  81     int              *vdwtype;
  82     real             *vdwparam;
  83     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
  84     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
  85     __m128i          vfitab;
  86     __m128i          ifour       = _mm_set1_epi32(4);
  87     __m128d          rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
  88     real             *vftab;
  89     __m128d          dummy_mask,cutoff_mask;
  90     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
  91     __m128d          one     = _mm_set1_pd(1.0);
  92     __m128d          two     = _mm_set1_pd(2.0);
  93     x                = xx[0];
  94     f                = ff[0];
  95
  96     nri              = nlist->nri;
  97     iinr             = nlist->iinr;
  98     jindex           = nlist->jindex;
  99     jjnr             = nlist->jjnr;
 100     shiftidx         = nlist->shift;
 101     gid              = nlist->gid;
 102     shiftvec         = fr->shift_vec[0];
 103     fshift           = fr->fshift[0];
 104     facel            = _mm_set1_pd(fr->epsfac);
 105     charge           = mdatoms->chargeA;
 106     nvdwtype         = fr->ntype;
 107     vdwparam         = fr->nbfp;
 108     vdwtype          = mdatoms->typeA;
 109
 110     vftab            = kernel_data->table_elec_vdw->data;
 111     vftabscale       = _mm_set1_pd(kernel_data->table_elec_vdw->scale);
 112
 113     /* Setup water-specific parameters */
 114     inr              = nlist->iinr[0];
 115     iq0              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
 116     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
 117     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
 118     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 119
 120     /* Avoid stupid compiler warnings */
 121     jnrA = jnrB = 0;
 122     j_coord_offsetA = 0;
 123     j_coord_offsetB = 0;
 124
 125     outeriter        = 0;
 126     inneriter        = 0;
 127
 128     /* Start outer loop over neighborlists */
 129     for(iidx=0; iidx<nri; iidx++)
 130     {
 131         /* Load shift vector for this list */
 132         i_shift_offset   = DIM*shiftidx[iidx];
 133
 134         /* Load limits for loop over neighbors */
 135         j_index_start    = jindex[iidx];
 136         j_index_end      = jindex[iidx+1];
 137
 138         /* Get outer coordinate index */
 139         inr              = iinr[iidx];
 140         i_coord_offset   = DIM*inr;
 141
 142         /* Load i particle coords and add shift vector */
 143         gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
 144                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 145
 146         fix0             = _mm_setzero_pd();
 147         fiy0             = _mm_setzero_pd();
 148         fiz0             = _mm_setzero_pd();
 149         fix1             = _mm_setzero_pd();
 150         fiy1             = _mm_setzero_pd();
 151         fiz1             = _mm_setzero_pd();
 152         fix2             = _mm_setzero_pd();
 153         fiy2             = _mm_setzero_pd();
 154         fiz2             = _mm_setzero_pd();
 155
 156         /* Reset potential sums */
 157         velecsum         = _mm_setzero_pd();
 158         vvdwsum          = _mm_setzero_pd();
 159
 160         /* Start inner kernel loop */
 161         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
 162         {
 163
 164             /* Get j neighbor index, and coordinate index */
 165             jnrA             = jjnr[jidx];
 166             jnrB             = jjnr[jidx+1];
 167             j_coord_offsetA  = DIM*jnrA;
 168             j_coord_offsetB  = DIM*jnrB;
 169
 170             /* load j atom coordinates */
 171             gmx_mm_load_1rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
 172                                               &jx0,&jy0,&jz0);
 173
 174             /* Calculate displacement vector */
 175             dx00             = _mm_sub_pd(ix0,jx0);
 176             dy00             = _mm_sub_pd(iy0,jy0);
 177             dz00             = _mm_sub_pd(iz0,jz0);
 178             dx10             = _mm_sub_pd(ix1,jx0);
 179             dy10             = _mm_sub_pd(iy1,jy0);
 180             dz10             = _mm_sub_pd(iz1,jz0);
 181             dx20             = _mm_sub_pd(ix2,jx0);
 182             dy20             = _mm_sub_pd(iy2,jy0);
 183             dz20             = _mm_sub_pd(iz2,jz0);
 184
 185             /* Calculate squared distance and things based on it */
 186             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 187             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 188             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 189
 190             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 191             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 192             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 193
 194             /* Load parameters for j particles */
 195             jq0              = gmx_mm_load_2real_swizzle_pd(charge+jnrA+0,charge+jnrB+0);
 196             vdwjidx0A        = 2*vdwtype[jnrA+0];
 197             vdwjidx0B        = 2*vdwtype[jnrB+0];
 198
 199             fjx0             = _mm_setzero_pd();
 200             fjy0             = _mm_setzero_pd();
 201             fjz0             = _mm_setzero_pd();
 202
 203             /**************************
 204              * CALCULATE INTERACTIONS *
 205              **************************/
 206
 207             r00              = _mm_mul_pd(rsq00,rinv00);
 208
 209             /* Compute parameters for interactions between i and j atoms */
 210             qq00             = _mm_mul_pd(iq0,jq0);
 211             gmx_mm_load_2pair_swizzle_pd(vdwparam+vdwioffset0+vdwjidx0A,
 212                                          vdwparam+vdwioffset0+vdwjidx0B,&c6_00,&c12_00);
 213
 214             /* Calculate table index by multiplying r with table scale and truncate to integer */
 215             rt               = _mm_mul_pd(r00,vftabscale);
 216             vfitab           = _mm_cvttpd_epi32(rt);
 217             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 218             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 219
 220             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 221             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 222             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 223             GMX_MM_TRANSPOSE2_PD(Y,F);
 224             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 225             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 226             GMX_MM_TRANSPOSE2_PD(G,H);
 227             Heps             = _mm_mul_pd(vfeps,H);
 228             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 229             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 230             velec            = _mm_mul_pd(qq00,VV);
 231             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 232             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq00,FF),_mm_mul_pd(vftabscale,rinv00)));
 233
 234             /* CUBIC SPLINE TABLE DISPERSION */
 235             vfitab           = _mm_add_epi32(vfitab,ifour);
 236             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 237             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 238             GMX_MM_TRANSPOSE2_PD(Y,F);
 239             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 240             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 241             GMX_MM_TRANSPOSE2_PD(G,H);
 242             Heps             = _mm_mul_pd(vfeps,H);
 243             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 244             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 245             vvdw6            = _mm_mul_pd(c6_00,VV);
 246             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 247             fvdw6            = _mm_mul_pd(c6_00,FF);
 248
 249             /* CUBIC SPLINE TABLE REPULSION */
 250             vfitab           = _mm_add_epi32(vfitab,ifour);
 251             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 252             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 253             GMX_MM_TRANSPOSE2_PD(Y,F);
 254             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 255             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 256             GMX_MM_TRANSPOSE2_PD(G,H);
 257             Heps             = _mm_mul_pd(vfeps,H);
 258             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 259             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 260             vvdw12           = _mm_mul_pd(c12_00,VV);
 261             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 262             fvdw12           = _mm_mul_pd(c12_00,FF);
 263             vvdw             = _mm_add_pd(vvdw12,vvdw6);
 264             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
 265
 266             /* Update potential sum for this i atom from the interaction with this j atom. */
 267             velecsum         = _mm_add_pd(velecsum,velec);
 268             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 269
 270             fscal            = _mm_add_pd(felec,fvdw);
 271
 272             /* Calculate temporary vectorial force */
 273             tx               = _mm_mul_pd(fscal,dx00);
 274             ty               = _mm_mul_pd(fscal,dy00);
 275             tz               = _mm_mul_pd(fscal,dz00);
 276
 277             /* Update vectorial force */
 278             fix0             = _mm_add_pd(fix0,tx);
 279             fiy0             = _mm_add_pd(fiy0,ty);
 280             fiz0             = _mm_add_pd(fiz0,tz);
 281
 282             fjx0             = _mm_add_pd(fjx0,tx);
 283             fjy0             = _mm_add_pd(fjy0,ty);
 284             fjz0             = _mm_add_pd(fjz0,tz);
 285
 286             /**************************
 287              * CALCULATE INTERACTIONS *
 288              **************************/
 289
 290             r10              = _mm_mul_pd(rsq10,rinv10);
 291
 292             /* Compute parameters for interactions between i and j atoms */
 293             qq10             = _mm_mul_pd(iq1,jq0);
 294
 295             /* Calculate table index by multiplying r with table scale and truncate to integer */
 296             rt               = _mm_mul_pd(r10,vftabscale);
 297             vfitab           = _mm_cvttpd_epi32(rt);
 298             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 299             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 300
 301             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 302             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 303             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 304             GMX_MM_TRANSPOSE2_PD(Y,F);
 305             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 306             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 307             GMX_MM_TRANSPOSE2_PD(G,H);
 308             Heps             = _mm_mul_pd(vfeps,H);
 309             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 310             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 311             velec            = _mm_mul_pd(qq10,VV);
 312             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 313             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq10,FF),_mm_mul_pd(vftabscale,rinv10)));
 314
 315             /* Update potential sum for this i atom from the interaction with this j atom. */
 316             velecsum         = _mm_add_pd(velecsum,velec);
 317
 318             fscal            = felec;
 319
 320             /* Calculate temporary vectorial force */
 321             tx               = _mm_mul_pd(fscal,dx10);
 322             ty               = _mm_mul_pd(fscal,dy10);
 323             tz               = _mm_mul_pd(fscal,dz10);
 324
 325             /* Update vectorial force */
 326             fix1             = _mm_add_pd(fix1,tx);
 327             fiy1             = _mm_add_pd(fiy1,ty);
 328             fiz1             = _mm_add_pd(fiz1,tz);
 329
 330             fjx0             = _mm_add_pd(fjx0,tx);
 331             fjy0             = _mm_add_pd(fjy0,ty);
 332             fjz0             = _mm_add_pd(fjz0,tz);
 333
 334             /**************************
 335              * CALCULATE INTERACTIONS *
 336              **************************/
 337
 338             r20              = _mm_mul_pd(rsq20,rinv20);
 339
 340             /* Compute parameters for interactions between i and j atoms */
 341             qq20             = _mm_mul_pd(iq2,jq0);
 342
 343             /* Calculate table index by multiplying r with table scale and truncate to integer */
 344             rt               = _mm_mul_pd(r20,vftabscale);
 345             vfitab           = _mm_cvttpd_epi32(rt);
 346             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 347             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 348
 349             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 350             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 351             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 352             GMX_MM_TRANSPOSE2_PD(Y,F);
 353             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 354             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 355             GMX_MM_TRANSPOSE2_PD(G,H);
 356             Heps             = _mm_mul_pd(vfeps,H);
 357             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 358             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 359             velec            = _mm_mul_pd(qq20,VV);
 360             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 361             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq20,FF),_mm_mul_pd(vftabscale,rinv20)));
 362
 363             /* Update potential sum for this i atom from the interaction with this j atom. */
 364             velecsum         = _mm_add_pd(velecsum,velec);
 365
 366             fscal            = felec;
 367
 368             /* Calculate temporary vectorial force */
 369             tx               = _mm_mul_pd(fscal,dx20);
 370             ty               = _mm_mul_pd(fscal,dy20);
 371             tz               = _mm_mul_pd(fscal,dz20);
 372
 373             /* Update vectorial force */
 374             fix2             = _mm_add_pd(fix2,tx);
 375             fiy2             = _mm_add_pd(fiy2,ty);
 376             fiz2             = _mm_add_pd(fiz2,tz);
 377
 378             fjx0             = _mm_add_pd(fjx0,tx);
 379             fjy0             = _mm_add_pd(fjy0,ty);
 380             fjz0             = _mm_add_pd(fjz0,tz);
 381
 382             gmx_mm_decrement_1rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0);
 383
 384             /* Inner loop uses 162 flops */
 385         }
 386
 387         if(jidx<j_index_end)
 388         {
 389
 390             jnrA             = jjnr[jidx];
 391             j_coord_offsetA  = DIM*jnrA;
 392
 393             /* load j atom coordinates */
 394             gmx_mm_load_1rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
 395                                               &jx0,&jy0,&jz0);
 396
 397             /* Calculate displacement vector */
 398             dx00             = _mm_sub_pd(ix0,jx0);
 399             dy00             = _mm_sub_pd(iy0,jy0);
 400             dz00             = _mm_sub_pd(iz0,jz0);
 401             dx10             = _mm_sub_pd(ix1,jx0);
 402             dy10             = _mm_sub_pd(iy1,jy0);
 403             dz10             = _mm_sub_pd(iz1,jz0);
 404             dx20             = _mm_sub_pd(ix2,jx0);
 405             dy20             = _mm_sub_pd(iy2,jy0);
 406             dz20             = _mm_sub_pd(iz2,jz0);
 407
 408             /* Calculate squared distance and things based on it */
 409             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 410             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 411             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 412
 413             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 414             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 415             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 416
 417             /* Load parameters for j particles */
 418             jq0              = _mm_load_sd(charge+jnrA+0);
 419             vdwjidx0A        = 2*vdwtype[jnrA+0];
 420
 421             fjx0             = _mm_setzero_pd();
 422             fjy0             = _mm_setzero_pd();
 423             fjz0             = _mm_setzero_pd();
 424
 425             /**************************
 426              * CALCULATE INTERACTIONS *
 427              **************************/
 428
 429             r00              = _mm_mul_pd(rsq00,rinv00);
 430
 431             /* Compute parameters for interactions between i and j atoms */
 432             qq00             = _mm_mul_pd(iq0,jq0);
 433             gmx_mm_load_1pair_swizzle_pd(vdwparam+vdwioffset0+vdwjidx0A,&c6_00,&c12_00);
 434
 435             /* Calculate table index by multiplying r with table scale and truncate to integer */
 436             rt               = _mm_mul_pd(r00,vftabscale);
 437             vfitab           = _mm_cvttpd_epi32(rt);
 438             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 439             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 440
 441             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 442             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 443             F                = _mm_setzero_pd();
 444             GMX_MM_TRANSPOSE2_PD(Y,F);
 445             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 446             H                = _mm_setzero_pd();
 447             GMX_MM_TRANSPOSE2_PD(G,H);
 448             Heps             = _mm_mul_pd(vfeps,H);
 449             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 450             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 451             velec            = _mm_mul_pd(qq00,VV);
 452             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 453             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq00,FF),_mm_mul_pd(vftabscale,rinv00)));
 454
 455             /* CUBIC SPLINE TABLE DISPERSION */
 456             vfitab           = _mm_add_epi32(vfitab,ifour);
 457             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 458             F                = _mm_setzero_pd();
 459             GMX_MM_TRANSPOSE2_PD(Y,F);
 460             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 461             H                = _mm_setzero_pd();
 462             GMX_MM_TRANSPOSE2_PD(G,H);
 463             Heps             = _mm_mul_pd(vfeps,H);
 464             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 465             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 466             vvdw6            = _mm_mul_pd(c6_00,VV);
 467             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 468             fvdw6            = _mm_mul_pd(c6_00,FF);
 469
 470             /* CUBIC SPLINE TABLE REPULSION */
 471             vfitab           = _mm_add_epi32(vfitab,ifour);
 472             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 473             F                = _mm_setzero_pd();
 474             GMX_MM_TRANSPOSE2_PD(Y,F);
 475             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 476             H                = _mm_setzero_pd();
 477             GMX_MM_TRANSPOSE2_PD(G,H);
 478             Heps             = _mm_mul_pd(vfeps,H);
 479             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 480             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 481             vvdw12           = _mm_mul_pd(c12_00,VV);
 482             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 483             fvdw12           = _mm_mul_pd(c12_00,FF);
 484             vvdw             = _mm_add_pd(vvdw12,vvdw6);
 485             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
 486
 487             /* Update potential sum for this i atom from the interaction with this j atom. */
 488             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 489             velecsum         = _mm_add_pd(velecsum,velec);
 490             vvdw             = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
 491             vvdwsum          = _mm_add_pd(vvdwsum,vvdw);
 492
 493             fscal            = _mm_add_pd(felec,fvdw);
 494
 495             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 496
 497             /* Calculate temporary vectorial force */
 498             tx               = _mm_mul_pd(fscal,dx00);
 499             ty               = _mm_mul_pd(fscal,dy00);
 500             tz               = _mm_mul_pd(fscal,dz00);
 501
 502             /* Update vectorial force */
 503             fix0             = _mm_add_pd(fix0,tx);
 504             fiy0             = _mm_add_pd(fiy0,ty);
 505             fiz0             = _mm_add_pd(fiz0,tz);
 506
 507             fjx0             = _mm_add_pd(fjx0,tx);
 508             fjy0             = _mm_add_pd(fjy0,ty);
 509             fjz0             = _mm_add_pd(fjz0,tz);
 510
 511             /**************************
 512              * CALCULATE INTERACTIONS *
 513              **************************/
 514
 515             r10              = _mm_mul_pd(rsq10,rinv10);
 516
 517             /* Compute parameters for interactions between i and j atoms */
 518             qq10             = _mm_mul_pd(iq1,jq0);
 519
 520             /* Calculate table index by multiplying r with table scale and truncate to integer */
 521             rt               = _mm_mul_pd(r10,vftabscale);
 522             vfitab           = _mm_cvttpd_epi32(rt);
 523             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 524             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 525
 526             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 527             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 528             F                = _mm_setzero_pd();
 529             GMX_MM_TRANSPOSE2_PD(Y,F);
 530             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 531             H                = _mm_setzero_pd();
 532             GMX_MM_TRANSPOSE2_PD(G,H);
 533             Heps             = _mm_mul_pd(vfeps,H);
 534             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 535             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 536             velec            = _mm_mul_pd(qq10,VV);
 537             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 538             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq10,FF),_mm_mul_pd(vftabscale,rinv10)));
 539
 540             /* Update potential sum for this i atom from the interaction with this j atom. */
 541             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 542             velecsum         = _mm_add_pd(velecsum,velec);
 543
 544             fscal            = felec;
 545
 546             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 547
 548             /* Calculate temporary vectorial force */
 549             tx               = _mm_mul_pd(fscal,dx10);
 550             ty               = _mm_mul_pd(fscal,dy10);
 551             tz               = _mm_mul_pd(fscal,dz10);
 552
 553             /* Update vectorial force */
 554             fix1             = _mm_add_pd(fix1,tx);
 555             fiy1             = _mm_add_pd(fiy1,ty);
 556             fiz1             = _mm_add_pd(fiz1,tz);
 557
 558             fjx0             = _mm_add_pd(fjx0,tx);
 559             fjy0             = _mm_add_pd(fjy0,ty);
 560             fjz0             = _mm_add_pd(fjz0,tz);
 561
 562             /**************************
 563              * CALCULATE INTERACTIONS *
 564              **************************/
 565
 566             r20              = _mm_mul_pd(rsq20,rinv20);
 567
 568             /* Compute parameters for interactions between i and j atoms */
 569             qq20             = _mm_mul_pd(iq2,jq0);
 570
 571             /* Calculate table index by multiplying r with table scale and truncate to integer */
 572             rt               = _mm_mul_pd(r20,vftabscale);
 573             vfitab           = _mm_cvttpd_epi32(rt);
 574             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 575             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 576
 577             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 578             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 579             F                = _mm_setzero_pd();
 580             GMX_MM_TRANSPOSE2_PD(Y,F);
 581             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 582             H                = _mm_setzero_pd();
 583             GMX_MM_TRANSPOSE2_PD(G,H);
 584             Heps             = _mm_mul_pd(vfeps,H);
 585             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 586             VV               = _mm_add_pd(Y,_mm_mul_pd(vfeps,Fp));
 587             velec            = _mm_mul_pd(qq20,VV);
 588             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 589             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq20,FF),_mm_mul_pd(vftabscale,rinv20)));
 590
 591             /* Update potential sum for this i atom from the interaction with this j atom. */
 592             velec            = _mm_unpacklo_pd(velec,_mm_setzero_pd());
 593             velecsum         = _mm_add_pd(velecsum,velec);
 594
 595             fscal            = felec;
 596
 597             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
 598
 599             /* Calculate temporary vectorial force */
 600             tx               = _mm_mul_pd(fscal,dx20);
 601             ty               = _mm_mul_pd(fscal,dy20);
 602             tz               = _mm_mul_pd(fscal,dz20);
 603
 604             /* Update vectorial force */
 605             fix2             = _mm_add_pd(fix2,tx);
 606             fiy2             = _mm_add_pd(fiy2,ty);
 607             fiz2             = _mm_add_pd(fiz2,tz);
 608
 609             fjx0             = _mm_add_pd(fjx0,tx);
 610             fjy0             = _mm_add_pd(fjy0,ty);
 611             fjz0             = _mm_add_pd(fjz0,tz);
 612
 613             gmx_mm_decrement_1rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0);
 614
 615             /* Inner loop uses 162 flops */
 616         }
 617
 618         /* End of innermost loop */
 619
 620         gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
 621                                               f+i_coord_offset,fshift+i_shift_offset);
 622
 623         ggid                        = gid[iidx];
 624         /* Update potential energies */
 625         gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
 626         gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
 627
 628         /* Increment number of inner iterations */
 629         inneriter                  += j_index_end - j_index_start;
 630
 631         /* Outer loop uses 20 flops */
 632     }
 633
 634     /* Increment number of outer iterations */
 635     outeriter        += nri;
 636
 637     /* Update outer/inner flops */
 638
 639     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3_VF,outeriter*20 + inneriter*162);
 640 }
 641 /*
 642  * Gromacs nonbonded kernel:   nb_kernel_ElecCSTab_VdwCSTab_GeomW3P1_F_sse4_1_double
 643  * Electrostatics interaction: CubicSplineTable
 644  * VdW interaction:            CubicSplineTable
 645  * Geometry:                   Water3-Particle
 646  * Calculate force/pot:        Force
 647  */
 648 void
 649 nb_kernel_ElecCSTab_VdwCSTab_GeomW3P1_F_sse4_1_double
 650                     (t_nblist * gmx_restrict                nlist,
 651                      rvec * gmx_restrict                    xx,
 652                      rvec * gmx_restrict                    ff,
 653                      t_forcerec * gmx_restrict              fr,
 654                      t_mdatoms * gmx_restrict               mdatoms,
 655                      nb_kernel_data_t * gmx_restrict        kernel_data,
 656                      t_nrnb * gmx_restrict                  nrnb)
 657 {
 658     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 659      * just 0 for non-waters.
 660      * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
 661      * jnr indices corresponding to data put in the four positions in the SIMD register.
 662      */
 663     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 664     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 665     int              jnrA,jnrB;
 666     int              j_coord_offsetA,j_coord_offsetB;
 667     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 668     real             rcutoff_scalar;
 669     real             *shiftvec,*fshift,*x,*f;
 670     __m128d          tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
 671     int              vdwioffset0;
 672     __m128d          ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 673     int              vdwioffset1;
 674     __m128d          ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 675     int              vdwioffset2;
 676     __m128d          ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 677     int              vdwjidx0A,vdwjidx0B;
 678     __m128d          jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 679     __m128d          dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 680     __m128d          dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 681     __m128d          dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 682     __m128d          velec,felec,velecsum,facel,crf,krf,krf2;
 683     real             *charge;
 684     int              nvdwtype;
 685     __m128d          rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 686     int              *vdwtype;
 687     real             *vdwparam;
 688     __m128d          one_sixth   = _mm_set1_pd(1.0/6.0);
 689     __m128d          one_twelfth = _mm_set1_pd(1.0/12.0);
 690     __m128i          vfitab;
 691     __m128i          ifour       = _mm_set1_epi32(4);
 692     __m128d          rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
 693     real             *vftab;
 694     __m128d          dummy_mask,cutoff_mask;
 695     __m128d          signbit   = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
 696     __m128d          one     = _mm_set1_pd(1.0);
 697     __m128d          two     = _mm_set1_pd(2.0);
 698     x                = xx[0];
 699     f                = ff[0];
 700
 701     nri              = nlist->nri;
 702     iinr             = nlist->iinr;
 703     jindex           = nlist->jindex;
 704     jjnr             = nlist->jjnr;
 705     shiftidx         = nlist->shift;
 706     gid              = nlist->gid;
 707     shiftvec         = fr->shift_vec[0];
 708     fshift           = fr->fshift[0];
 709     facel            = _mm_set1_pd(fr->epsfac);
 710     charge           = mdatoms->chargeA;
 711     nvdwtype         = fr->ntype;
 712     vdwparam         = fr->nbfp;
 713     vdwtype          = mdatoms->typeA;
 714
 715     vftab            = kernel_data->table_elec_vdw->data;
 716     vftabscale       = _mm_set1_pd(kernel_data->table_elec_vdw->scale);
 717
 718     /* Setup water-specific parameters */
 719     inr              = nlist->iinr[0];
 720     iq0              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
 721     iq1              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
 722     iq2              = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
 723     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 724
 725     /* Avoid stupid compiler warnings */
 726     jnrA = jnrB = 0;
 727     j_coord_offsetA = 0;
 728     j_coord_offsetB = 0;
 729
 730     outeriter        = 0;
 731     inneriter        = 0;
 732
 733     /* Start outer loop over neighborlists */
 734     for(iidx=0; iidx<nri; iidx++)
 735     {
 736         /* Load shift vector for this list */
 737         i_shift_offset   = DIM*shiftidx[iidx];
 738
 739         /* Load limits for loop over neighbors */
 740         j_index_start    = jindex[iidx];
 741         j_index_end      = jindex[iidx+1];
 742
 743         /* Get outer coordinate index */
 744         inr              = iinr[iidx];
 745         i_coord_offset   = DIM*inr;
 746
 747         /* Load i particle coords and add shift vector */
 748         gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
 749                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 750
 751         fix0             = _mm_setzero_pd();
 752         fiy0             = _mm_setzero_pd();
 753         fiz0             = _mm_setzero_pd();
 754         fix1             = _mm_setzero_pd();
 755         fiy1             = _mm_setzero_pd();
 756         fiz1             = _mm_setzero_pd();
 757         fix2             = _mm_setzero_pd();
 758         fiy2             = _mm_setzero_pd();
 759         fiz2             = _mm_setzero_pd();
 760
 761         /* Start inner kernel loop */
 762         for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
 763         {
 764
 765             /* Get j neighbor index, and coordinate index */
 766             jnrA             = jjnr[jidx];
 767             jnrB             = jjnr[jidx+1];
 768             j_coord_offsetA  = DIM*jnrA;
 769             j_coord_offsetB  = DIM*jnrB;
 770
 771             /* load j atom coordinates */
 772             gmx_mm_load_1rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
 773                                               &jx0,&jy0,&jz0);
 774
 775             /* Calculate displacement vector */
 776             dx00             = _mm_sub_pd(ix0,jx0);
 777             dy00             = _mm_sub_pd(iy0,jy0);
 778             dz00             = _mm_sub_pd(iz0,jz0);
 779             dx10             = _mm_sub_pd(ix1,jx0);
 780             dy10             = _mm_sub_pd(iy1,jy0);
 781             dz10             = _mm_sub_pd(iz1,jz0);
 782             dx20             = _mm_sub_pd(ix2,jx0);
 783             dy20             = _mm_sub_pd(iy2,jy0);
 784             dz20             = _mm_sub_pd(iz2,jz0);
 785
 786             /* Calculate squared distance and things based on it */
 787             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 788             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 789             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 790
 791             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 792             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 793             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 794
 795             /* Load parameters for j particles */
 796             jq0              = gmx_mm_load_2real_swizzle_pd(charge+jnrA+0,charge+jnrB+0);
 797             vdwjidx0A        = 2*vdwtype[jnrA+0];
 798             vdwjidx0B        = 2*vdwtype[jnrB+0];
 799
 800             fjx0             = _mm_setzero_pd();
 801             fjy0             = _mm_setzero_pd();
 802             fjz0             = _mm_setzero_pd();
 803
 804             /**************************
 805              * CALCULATE INTERACTIONS *
 806              **************************/
 807
 808             r00              = _mm_mul_pd(rsq00,rinv00);
 809
 810             /* Compute parameters for interactions between i and j atoms */
 811             qq00             = _mm_mul_pd(iq0,jq0);
 812             gmx_mm_load_2pair_swizzle_pd(vdwparam+vdwioffset0+vdwjidx0A,
 813                                          vdwparam+vdwioffset0+vdwjidx0B,&c6_00,&c12_00);
 814
 815             /* Calculate table index by multiplying r with table scale and truncate to integer */
 816             rt               = _mm_mul_pd(r00,vftabscale);
 817             vfitab           = _mm_cvttpd_epi32(rt);
 818             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 819             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 820
 821             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 822             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 823             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 824             GMX_MM_TRANSPOSE2_PD(Y,F);
 825             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 826             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 827             GMX_MM_TRANSPOSE2_PD(G,H);
 828             Heps             = _mm_mul_pd(vfeps,H);
 829             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 830             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 831             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq00,FF),_mm_mul_pd(vftabscale,rinv00)));
 832
 833             /* CUBIC SPLINE TABLE DISPERSION */
 834             vfitab           = _mm_add_epi32(vfitab,ifour);
 835             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 836             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 837             GMX_MM_TRANSPOSE2_PD(Y,F);
 838             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 839             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 840             GMX_MM_TRANSPOSE2_PD(G,H);
 841             Heps             = _mm_mul_pd(vfeps,H);
 842             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 843             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 844             fvdw6            = _mm_mul_pd(c6_00,FF);
 845
 846             /* CUBIC SPLINE TABLE REPULSION */
 847             vfitab           = _mm_add_epi32(vfitab,ifour);
 848             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 849             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 850             GMX_MM_TRANSPOSE2_PD(Y,F);
 851             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 852             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 853             GMX_MM_TRANSPOSE2_PD(G,H);
 854             Heps             = _mm_mul_pd(vfeps,H);
 855             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 856             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 857             fvdw12           = _mm_mul_pd(c12_00,FF);
 858             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
 859
 860             fscal            = _mm_add_pd(felec,fvdw);
 861
 862             /* Calculate temporary vectorial force */
 863             tx               = _mm_mul_pd(fscal,dx00);
 864             ty               = _mm_mul_pd(fscal,dy00);
 865             tz               = _mm_mul_pd(fscal,dz00);
 866
 867             /* Update vectorial force */
 868             fix0             = _mm_add_pd(fix0,tx);
 869             fiy0             = _mm_add_pd(fiy0,ty);
 870             fiz0             = _mm_add_pd(fiz0,tz);
 871
 872             fjx0             = _mm_add_pd(fjx0,tx);
 873             fjy0             = _mm_add_pd(fjy0,ty);
 874             fjz0             = _mm_add_pd(fjz0,tz);
 875
 876             /**************************
 877              * CALCULATE INTERACTIONS *
 878              **************************/
 879
 880             r10              = _mm_mul_pd(rsq10,rinv10);
 881
 882             /* Compute parameters for interactions between i and j atoms */
 883             qq10             = _mm_mul_pd(iq1,jq0);
 884
 885             /* Calculate table index by multiplying r with table scale and truncate to integer */
 886             rt               = _mm_mul_pd(r10,vftabscale);
 887             vfitab           = _mm_cvttpd_epi32(rt);
 888             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 889             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 890
 891             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 892             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 893             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 894             GMX_MM_TRANSPOSE2_PD(Y,F);
 895             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 896             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 897             GMX_MM_TRANSPOSE2_PD(G,H);
 898             Heps             = _mm_mul_pd(vfeps,H);
 899             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 900             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 901             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq10,FF),_mm_mul_pd(vftabscale,rinv10)));
 902
 903             fscal            = felec;
 904
 905             /* Calculate temporary vectorial force */
 906             tx               = _mm_mul_pd(fscal,dx10);
 907             ty               = _mm_mul_pd(fscal,dy10);
 908             tz               = _mm_mul_pd(fscal,dz10);
 909
 910             /* Update vectorial force */
 911             fix1             = _mm_add_pd(fix1,tx);
 912             fiy1             = _mm_add_pd(fiy1,ty);
 913             fiz1             = _mm_add_pd(fiz1,tz);
 914
 915             fjx0             = _mm_add_pd(fjx0,tx);
 916             fjy0             = _mm_add_pd(fjy0,ty);
 917             fjz0             = _mm_add_pd(fjz0,tz);
 918
 919             /**************************
 920              * CALCULATE INTERACTIONS *
 921              **************************/
 922
 923             r20              = _mm_mul_pd(rsq20,rinv20);
 924
 925             /* Compute parameters for interactions between i and j atoms */
 926             qq20             = _mm_mul_pd(iq2,jq0);
 927
 928             /* Calculate table index by multiplying r with table scale and truncate to integer */
 929             rt               = _mm_mul_pd(r20,vftabscale);
 930             vfitab           = _mm_cvttpd_epi32(rt);
 931             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
 932             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
 933
 934             /* CUBIC SPLINE TABLE ELECTROSTATICS */
 935             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
 936             F                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) );
 937             GMX_MM_TRANSPOSE2_PD(Y,F);
 938             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
 939             H                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,1) +2);
 940             GMX_MM_TRANSPOSE2_PD(G,H);
 941             Heps             = _mm_mul_pd(vfeps,H);
 942             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
 943             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
 944             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq20,FF),_mm_mul_pd(vftabscale,rinv20)));
 945
 946             fscal            = felec;
 947
 948             /* Calculate temporary vectorial force */
 949             tx               = _mm_mul_pd(fscal,dx20);
 950             ty               = _mm_mul_pd(fscal,dy20);
 951             tz               = _mm_mul_pd(fscal,dz20);
 952
 953             /* Update vectorial force */
 954             fix2             = _mm_add_pd(fix2,tx);
 955             fiy2             = _mm_add_pd(fiy2,ty);
 956             fiz2             = _mm_add_pd(fiz2,tz);
 957
 958             fjx0             = _mm_add_pd(fjx0,tx);
 959             fjy0             = _mm_add_pd(fjy0,ty);
 960             fjz0             = _mm_add_pd(fjz0,tz);
 961
 962             gmx_mm_decrement_1rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0);
 963
 964             /* Inner loop uses 142 flops */
 965         }
 966
 967         if(jidx<j_index_end)
 968         {
 969
 970             jnrA             = jjnr[jidx];
 971             j_coord_offsetA  = DIM*jnrA;
 972
 973             /* load j atom coordinates */
 974             gmx_mm_load_1rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
 975                                               &jx0,&jy0,&jz0);
 976
 977             /* Calculate displacement vector */
 978             dx00             = _mm_sub_pd(ix0,jx0);
 979             dy00             = _mm_sub_pd(iy0,jy0);
 980             dz00             = _mm_sub_pd(iz0,jz0);
 981             dx10             = _mm_sub_pd(ix1,jx0);
 982             dy10             = _mm_sub_pd(iy1,jy0);
 983             dz10             = _mm_sub_pd(iz1,jz0);
 984             dx20             = _mm_sub_pd(ix2,jx0);
 985             dy20             = _mm_sub_pd(iy2,jy0);
 986             dz20             = _mm_sub_pd(iz2,jz0);
 987
 988             /* Calculate squared distance and things based on it */
 989             rsq00            = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
 990             rsq10            = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
 991             rsq20            = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
 992
 993             rinv00           = gmx_mm_invsqrt_pd(rsq00);
 994             rinv10           = gmx_mm_invsqrt_pd(rsq10);
 995             rinv20           = gmx_mm_invsqrt_pd(rsq20);
 996
 997             /* Load parameters for j particles */
 998             jq0              = _mm_load_sd(charge+jnrA+0);
 999             vdwjidx0A        = 2*vdwtype[jnrA+0];
1000
1001             fjx0             = _mm_setzero_pd();
1002             fjy0             = _mm_setzero_pd();
1003             fjz0             = _mm_setzero_pd();
1004
1005             /**************************
1006              * CALCULATE INTERACTIONS *
1007              **************************/
1008
1009             r00              = _mm_mul_pd(rsq00,rinv00);
1010
1011             /* Compute parameters for interactions between i and j atoms */
1012             qq00             = _mm_mul_pd(iq0,jq0);
1013             gmx_mm_load_1pair_swizzle_pd(vdwparam+vdwioffset0+vdwjidx0A,&c6_00,&c12_00);
1014
1015             /* Calculate table index by multiplying r with table scale and truncate to integer */
1016             rt               = _mm_mul_pd(r00,vftabscale);
1017             vfitab           = _mm_cvttpd_epi32(rt);
1018             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1019             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1020
1021             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1022             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
1023             F                = _mm_setzero_pd();
1024             GMX_MM_TRANSPOSE2_PD(Y,F);
1025             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
1026             H                = _mm_setzero_pd();
1027             GMX_MM_TRANSPOSE2_PD(G,H);
1028             Heps             = _mm_mul_pd(vfeps,H);
1029             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
1030             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
1031             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq00,FF),_mm_mul_pd(vftabscale,rinv00)));
1032
1033             /* CUBIC SPLINE TABLE DISPERSION */
1034             vfitab           = _mm_add_epi32(vfitab,ifour);
1035             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
1036             F                = _mm_setzero_pd();
1037             GMX_MM_TRANSPOSE2_PD(Y,F);
1038             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
1039             H                = _mm_setzero_pd();
1040             GMX_MM_TRANSPOSE2_PD(G,H);
1041             Heps             = _mm_mul_pd(vfeps,H);
1042             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
1043             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
1044             fvdw6            = _mm_mul_pd(c6_00,FF);
1045
1046             /* CUBIC SPLINE TABLE REPULSION */
1047             vfitab           = _mm_add_epi32(vfitab,ifour);
1048             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
1049             F                = _mm_setzero_pd();
1050             GMX_MM_TRANSPOSE2_PD(Y,F);
1051             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
1052             H                = _mm_setzero_pd();
1053             GMX_MM_TRANSPOSE2_PD(G,H);
1054             Heps             = _mm_mul_pd(vfeps,H);
1055             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
1056             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
1057             fvdw12           = _mm_mul_pd(c12_00,FF);
1058             fvdw             = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
1059
1060             fscal            = _mm_add_pd(felec,fvdw);
1061
1062             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1063
1064             /* Calculate temporary vectorial force */
1065             tx               = _mm_mul_pd(fscal,dx00);
1066             ty               = _mm_mul_pd(fscal,dy00);
1067             tz               = _mm_mul_pd(fscal,dz00);
1068
1069             /* Update vectorial force */
1070             fix0             = _mm_add_pd(fix0,tx);
1071             fiy0             = _mm_add_pd(fiy0,ty);
1072             fiz0             = _mm_add_pd(fiz0,tz);
1073
1074             fjx0             = _mm_add_pd(fjx0,tx);
1075             fjy0             = _mm_add_pd(fjy0,ty);
1076             fjz0             = _mm_add_pd(fjz0,tz);
1077
1078             /**************************
1079              * CALCULATE INTERACTIONS *
1080              **************************/
1081
1082             r10              = _mm_mul_pd(rsq10,rinv10);
1083
1084             /* Compute parameters for interactions between i and j atoms */
1085             qq10             = _mm_mul_pd(iq1,jq0);
1086
1087             /* Calculate table index by multiplying r with table scale and truncate to integer */
1088             rt               = _mm_mul_pd(r10,vftabscale);
1089             vfitab           = _mm_cvttpd_epi32(rt);
1090             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1091             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1092
1093             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1094             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
1095             F                = _mm_setzero_pd();
1096             GMX_MM_TRANSPOSE2_PD(Y,F);
1097             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
1098             H                = _mm_setzero_pd();
1099             GMX_MM_TRANSPOSE2_PD(G,H);
1100             Heps             = _mm_mul_pd(vfeps,H);
1101             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
1102             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
1103             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq10,FF),_mm_mul_pd(vftabscale,rinv10)));
1104
1105             fscal            = felec;
1106
1107             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1108
1109             /* Calculate temporary vectorial force */
1110             tx               = _mm_mul_pd(fscal,dx10);
1111             ty               = _mm_mul_pd(fscal,dy10);
1112             tz               = _mm_mul_pd(fscal,dz10);
1113
1114             /* Update vectorial force */
1115             fix1             = _mm_add_pd(fix1,tx);
1116             fiy1             = _mm_add_pd(fiy1,ty);
1117             fiz1             = _mm_add_pd(fiz1,tz);
1118
1119             fjx0             = _mm_add_pd(fjx0,tx);
1120             fjy0             = _mm_add_pd(fjy0,ty);
1121             fjz0             = _mm_add_pd(fjz0,tz);
1122
1123             /**************************
1124              * CALCULATE INTERACTIONS *
1125              **************************/
1126
1127             r20              = _mm_mul_pd(rsq20,rinv20);
1128
1129             /* Compute parameters for interactions between i and j atoms */
1130             qq20             = _mm_mul_pd(iq2,jq0);
1131
1132             /* Calculate table index by multiplying r with table scale and truncate to integer */
1133             rt               = _mm_mul_pd(r20,vftabscale);
1134             vfitab           = _mm_cvttpd_epi32(rt);
1135             vfeps            = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1136             vfitab           = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1137
1138             /* CUBIC SPLINE TABLE ELECTROSTATICS */
1139             Y                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) );
1140             F                = _mm_setzero_pd();
1141             GMX_MM_TRANSPOSE2_PD(Y,F);
1142             G                = _mm_load_pd( vftab + gmx_mm_extract_epi32(vfitab,0) +2);
1143             H                = _mm_setzero_pd();
1144             GMX_MM_TRANSPOSE2_PD(G,H);
1145             Heps             = _mm_mul_pd(vfeps,H);
1146             Fp               = _mm_add_pd(F,_mm_mul_pd(vfeps,_mm_add_pd(G,Heps)));
1147             FF               = _mm_add_pd(Fp,_mm_mul_pd(vfeps,_mm_add_pd(G,_mm_add_pd(Heps,Heps))));
1148             felec            = _mm_xor_pd(signbit,_mm_mul_pd(_mm_mul_pd(qq20,FF),_mm_mul_pd(vftabscale,rinv20)));
1149
1150             fscal            = felec;
1151
1152             fscal            = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1153
1154             /* Calculate temporary vectorial force */
1155             tx               = _mm_mul_pd(fscal,dx20);
1156             ty               = _mm_mul_pd(fscal,dy20);
1157             tz               = _mm_mul_pd(fscal,dz20);
1158
1159             /* Update vectorial force */
1160             fix2             = _mm_add_pd(fix2,tx);
1161             fiy2             = _mm_add_pd(fiy2,ty);
1162             fiz2             = _mm_add_pd(fiz2,tz);
1163
1164             fjx0             = _mm_add_pd(fjx0,tx);
1165             fjy0             = _mm_add_pd(fjy0,ty);
1166             fjz0             = _mm_add_pd(fjz0,tz);
1167
1168             gmx_mm_decrement_1rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0);
1169
1170             /* Inner loop uses 142 flops */
1171         }
1172
1173         /* End of innermost loop */
1174
1175         gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1176                                               f+i_coord_offset,fshift+i_shift_offset);
1177
1178         /* Increment number of inner iterations */
1179         inneriter                  += j_index_end - j_index_start;
1180
1181         /* Outer loop uses 18 flops */
1182     }
1183
1184     /* Increment number of outer iterations */
1185     outeriter        += nri;
1186
1187     /* Update outer/inner flops */
1188
1189     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3_F,outeriter*18 + inneriter*142);
1190 }