src/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecCoul_VdwLJ_GeomW3W3_avx_128_fma_single.c

   1 /*
   2  * Note: this file was generated by the Gromacs avx_128_fma_single kernel generator.
   3  *
   4  *                This source code is part of
   5  *
   6  *                 G   R   O   M   A   C   S
   7  *
   8  * Copyright (c) 2001-2012, The GROMACS Development Team
   9  *
  10  * Gromacs is a library for molecular simulation and trajectory analysis,
  11  * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
  12  * a full list of developers and information, check out http://www.gromacs.org
  13  *
  14  * This program is free software; you can redistribute it and/or modify it under
  15  * the terms of the GNU Lesser General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option) any
  17  * later version.
  18  *
  19  * To help fund GROMACS development, we humbly ask that you cite
  20  * the papers people have written on it - you can find them on the website.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include <config.h>
  24 #endif
  25
  26 #include <math.h>
  27
  28 #include "../nb_kernel.h"
  29 #include "types/simple.h"
  30 #include "vec.h"
  31 #include "nrnb.h"
  32
  33 #include "gmx_math_x86_avx_128_fma_single.h"
  34 #include "kernelutil_x86_avx_128_fma_single.h"
  35
  36 /*
  37  * Gromacs nonbonded kernel:   nb_kernel_ElecCoul_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  38  * Electrostatics interaction: Coulomb
  39  * VdW interaction:            LennardJones
  40  * Geometry:                   Water3-Water3
  41  * Calculate force/pot:        PotentialAndForce
  42  */
  43 void
  44 nb_kernel_ElecCoul_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  45                     (t_nblist * gmx_restrict                nlist,
  46                      rvec * gmx_restrict                    xx,
  47                      rvec * gmx_restrict                    ff,
  48                      t_forcerec * gmx_restrict              fr,
  49                      t_mdatoms * gmx_restrict               mdatoms,
  50                      nb_kernel_data_t * gmx_restrict        kernel_data,
  51                      t_nrnb * gmx_restrict                  nrnb)
  52 {
  53     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  54      * just 0 for non-waters.
  55      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  56      * jnr indices corresponding to data put in the four positions in the SIMD register.
  57      */
  58     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  59     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  60     int              jnrA,jnrB,jnrC,jnrD;
  61     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  62     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  63     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  64     real             rcutoff_scalar;
  65     real             *shiftvec,*fshift,*x,*f;
  66     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  67     real             scratch[4*DIM];
  68     __m128           fscal,rcutoff,rcutoff2,jidxall;
  69     int              vdwioffset0;
  70     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  71     int              vdwioffset1;
  72     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  73     int              vdwioffset2;
  74     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  75     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  76     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  77     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
  78     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  79     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
  80     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  81     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  82     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
  83     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
  84     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  85     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
  86     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
  87     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
  88     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
  89     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
  90     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
  91     real             *charge;
  92     int              nvdwtype;
  93     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
  94     int              *vdwtype;
  95     real             *vdwparam;
  96     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
  97     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
  98     __m128           dummy_mask,cutoff_mask;
  99     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 100     __m128           one     = _mm_set1_ps(1.0);
 101     __m128           two     = _mm_set1_ps(2.0);
 102     x                = xx[0];
 103     f                = ff[0];
 104
 105     nri              = nlist->nri;
 106     iinr             = nlist->iinr;
 107     jindex           = nlist->jindex;
 108     jjnr             = nlist->jjnr;
 109     shiftidx         = nlist->shift;
 110     gid              = nlist->gid;
 111     shiftvec         = fr->shift_vec[0];
 112     fshift           = fr->fshift[0];
 113     facel            = _mm_set1_ps(fr->epsfac);
 114     charge           = mdatoms->chargeA;
 115     nvdwtype         = fr->ntype;
 116     vdwparam         = fr->nbfp;
 117     vdwtype          = mdatoms->typeA;
 118
 119     /* Setup water-specific parameters */
 120     inr              = nlist->iinr[0];
 121     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 122     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 123     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 124     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 125
 126     jq0              = _mm_set1_ps(charge[inr+0]);
 127     jq1              = _mm_set1_ps(charge[inr+1]);
 128     jq2              = _mm_set1_ps(charge[inr+2]);
 129     vdwjidx0A        = 2*vdwtype[inr+0];
 130     qq00             = _mm_mul_ps(iq0,jq0);
 131     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 132     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 133     qq01             = _mm_mul_ps(iq0,jq1);
 134     qq02             = _mm_mul_ps(iq0,jq2);
 135     qq10             = _mm_mul_ps(iq1,jq0);
 136     qq11             = _mm_mul_ps(iq1,jq1);
 137     qq12             = _mm_mul_ps(iq1,jq2);
 138     qq20             = _mm_mul_ps(iq2,jq0);
 139     qq21             = _mm_mul_ps(iq2,jq1);
 140     qq22             = _mm_mul_ps(iq2,jq2);
 141
 142     /* Avoid stupid compiler warnings */
 143     jnrA = jnrB = jnrC = jnrD = 0;
 144     j_coord_offsetA = 0;
 145     j_coord_offsetB = 0;
 146     j_coord_offsetC = 0;
 147     j_coord_offsetD = 0;
 148
 149     outeriter        = 0;
 150     inneriter        = 0;
 151
 152     for(iidx=0;iidx<4*DIM;iidx++)
 153     {
 154         scratch[iidx] = 0.0;
 155     }
 156
 157     /* Start outer loop over neighborlists */
 158     for(iidx=0; iidx<nri; iidx++)
 159     {
 160         /* Load shift vector for this list */
 161         i_shift_offset   = DIM*shiftidx[iidx];
 162
 163         /* Load limits for loop over neighbors */
 164         j_index_start    = jindex[iidx];
 165         j_index_end      = jindex[iidx+1];
 166
 167         /* Get outer coordinate index */
 168         inr              = iinr[iidx];
 169         i_coord_offset   = DIM*inr;
 170
 171         /* Load i particle coords and add shift vector */
 172         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 173                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 174
 175         fix0             = _mm_setzero_ps();
 176         fiy0             = _mm_setzero_ps();
 177         fiz0             = _mm_setzero_ps();
 178         fix1             = _mm_setzero_ps();
 179         fiy1             = _mm_setzero_ps();
 180         fiz1             = _mm_setzero_ps();
 181         fix2             = _mm_setzero_ps();
 182         fiy2             = _mm_setzero_ps();
 183         fiz2             = _mm_setzero_ps();
 184
 185         /* Reset potential sums */
 186         velecsum         = _mm_setzero_ps();
 187         vvdwsum          = _mm_setzero_ps();
 188
 189         /* Start inner kernel loop */
 190         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 191         {
 192
 193             /* Get j neighbor index, and coordinate index */
 194             jnrA             = jjnr[jidx];
 195             jnrB             = jjnr[jidx+1];
 196             jnrC             = jjnr[jidx+2];
 197             jnrD             = jjnr[jidx+3];
 198             j_coord_offsetA  = DIM*jnrA;
 199             j_coord_offsetB  = DIM*jnrB;
 200             j_coord_offsetC  = DIM*jnrC;
 201             j_coord_offsetD  = DIM*jnrD;
 202
 203             /* load j atom coordinates */
 204             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 205                                               x+j_coord_offsetC,x+j_coord_offsetD,
 206                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 207
 208             /* Calculate displacement vector */
 209             dx00             = _mm_sub_ps(ix0,jx0);
 210             dy00             = _mm_sub_ps(iy0,jy0);
 211             dz00             = _mm_sub_ps(iz0,jz0);
 212             dx01             = _mm_sub_ps(ix0,jx1);
 213             dy01             = _mm_sub_ps(iy0,jy1);
 214             dz01             = _mm_sub_ps(iz0,jz1);
 215             dx02             = _mm_sub_ps(ix0,jx2);
 216             dy02             = _mm_sub_ps(iy0,jy2);
 217             dz02             = _mm_sub_ps(iz0,jz2);
 218             dx10             = _mm_sub_ps(ix1,jx0);
 219             dy10             = _mm_sub_ps(iy1,jy0);
 220             dz10             = _mm_sub_ps(iz1,jz0);
 221             dx11             = _mm_sub_ps(ix1,jx1);
 222             dy11             = _mm_sub_ps(iy1,jy1);
 223             dz11             = _mm_sub_ps(iz1,jz1);
 224             dx12             = _mm_sub_ps(ix1,jx2);
 225             dy12             = _mm_sub_ps(iy1,jy2);
 226             dz12             = _mm_sub_ps(iz1,jz2);
 227             dx20             = _mm_sub_ps(ix2,jx0);
 228             dy20             = _mm_sub_ps(iy2,jy0);
 229             dz20             = _mm_sub_ps(iz2,jz0);
 230             dx21             = _mm_sub_ps(ix2,jx1);
 231             dy21             = _mm_sub_ps(iy2,jy1);
 232             dz21             = _mm_sub_ps(iz2,jz1);
 233             dx22             = _mm_sub_ps(ix2,jx2);
 234             dy22             = _mm_sub_ps(iy2,jy2);
 235             dz22             = _mm_sub_ps(iz2,jz2);
 236
 237             /* Calculate squared distance and things based on it */
 238             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 239             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 240             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 241             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 242             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 243             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 244             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 245             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 246             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 247
 248             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 249             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 250             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 251             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 252             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 253             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 254             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 255             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 256             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 257
 258             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 259             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
 260             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
 261             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 262             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 263             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 264             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 265             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 266             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 267
 268             fjx0             = _mm_setzero_ps();
 269             fjy0             = _mm_setzero_ps();
 270             fjz0             = _mm_setzero_ps();
 271             fjx1             = _mm_setzero_ps();
 272             fjy1             = _mm_setzero_ps();
 273             fjz1             = _mm_setzero_ps();
 274             fjx2             = _mm_setzero_ps();
 275             fjy2             = _mm_setzero_ps();
 276             fjz2             = _mm_setzero_ps();
 277
 278             /**************************
 279              * CALCULATE INTERACTIONS *
 280              **************************/
 281
 282             /* COULOMB ELECTROSTATICS */
 283             velec            = _mm_mul_ps(qq00,rinv00);
 284             felec            = _mm_mul_ps(velec,rinvsq00);
 285
 286             /* LENNARD-JONES DISPERSION/REPULSION */
 287
 288             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 289             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 290             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 291             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 292             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 293
 294             /* Update potential sum for this i atom from the interaction with this j atom. */
 295             velecsum         = _mm_add_ps(velecsum,velec);
 296             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 297
 298             fscal            = _mm_add_ps(felec,fvdw);
 299
 300              /* Update vectorial force */
 301             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 302             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 303             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 304
 305             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 306             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 307             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 308
 309             /**************************
 310              * CALCULATE INTERACTIONS *
 311              **************************/
 312
 313             /* COULOMB ELECTROSTATICS */
 314             velec            = _mm_mul_ps(qq01,rinv01);
 315             felec            = _mm_mul_ps(velec,rinvsq01);
 316
 317             /* Update potential sum for this i atom from the interaction with this j atom. */
 318             velecsum         = _mm_add_ps(velecsum,velec);
 319
 320             fscal            = felec;
 321
 322              /* Update vectorial force */
 323             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 324             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 325             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 326
 327             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 328             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 329             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 330
 331             /**************************
 332              * CALCULATE INTERACTIONS *
 333              **************************/
 334
 335             /* COULOMB ELECTROSTATICS */
 336             velec            = _mm_mul_ps(qq02,rinv02);
 337             felec            = _mm_mul_ps(velec,rinvsq02);
 338
 339             /* Update potential sum for this i atom from the interaction with this j atom. */
 340             velecsum         = _mm_add_ps(velecsum,velec);
 341
 342             fscal            = felec;
 343
 344              /* Update vectorial force */
 345             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 346             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 347             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 348
 349             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 350             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 351             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 352
 353             /**************************
 354              * CALCULATE INTERACTIONS *
 355              **************************/
 356
 357             /* COULOMB ELECTROSTATICS */
 358             velec            = _mm_mul_ps(qq10,rinv10);
 359             felec            = _mm_mul_ps(velec,rinvsq10);
 360
 361             /* Update potential sum for this i atom from the interaction with this j atom. */
 362             velecsum         = _mm_add_ps(velecsum,velec);
 363
 364             fscal            = felec;
 365
 366              /* Update vectorial force */
 367             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 368             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 369             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 370
 371             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 372             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 373             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 374
 375             /**************************
 376              * CALCULATE INTERACTIONS *
 377              **************************/
 378
 379             /* COULOMB ELECTROSTATICS */
 380             velec            = _mm_mul_ps(qq11,rinv11);
 381             felec            = _mm_mul_ps(velec,rinvsq11);
 382
 383             /* Update potential sum for this i atom from the interaction with this j atom. */
 384             velecsum         = _mm_add_ps(velecsum,velec);
 385
 386             fscal            = felec;
 387
 388              /* Update vectorial force */
 389             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 390             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 391             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 392
 393             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 394             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 395             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 396
 397             /**************************
 398              * CALCULATE INTERACTIONS *
 399              **************************/
 400
 401             /* COULOMB ELECTROSTATICS */
 402             velec            = _mm_mul_ps(qq12,rinv12);
 403             felec            = _mm_mul_ps(velec,rinvsq12);
 404
 405             /* Update potential sum for this i atom from the interaction with this j atom. */
 406             velecsum         = _mm_add_ps(velecsum,velec);
 407
 408             fscal            = felec;
 409
 410              /* Update vectorial force */
 411             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 412             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 413             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 414
 415             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 416             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 417             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 418
 419             /**************************
 420              * CALCULATE INTERACTIONS *
 421              **************************/
 422
 423             /* COULOMB ELECTROSTATICS */
 424             velec            = _mm_mul_ps(qq20,rinv20);
 425             felec            = _mm_mul_ps(velec,rinvsq20);
 426
 427             /* Update potential sum for this i atom from the interaction with this j atom. */
 428             velecsum         = _mm_add_ps(velecsum,velec);
 429
 430             fscal            = felec;
 431
 432              /* Update vectorial force */
 433             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 434             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 435             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 436
 437             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 438             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 439             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 440
 441             /**************************
 442              * CALCULATE INTERACTIONS *
 443              **************************/
 444
 445             /* COULOMB ELECTROSTATICS */
 446             velec            = _mm_mul_ps(qq21,rinv21);
 447             felec            = _mm_mul_ps(velec,rinvsq21);
 448
 449             /* Update potential sum for this i atom from the interaction with this j atom. */
 450             velecsum         = _mm_add_ps(velecsum,velec);
 451
 452             fscal            = felec;
 453
 454              /* Update vectorial force */
 455             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 456             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 457             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 458
 459             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 460             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 461             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 462
 463             /**************************
 464              * CALCULATE INTERACTIONS *
 465              **************************/
 466
 467             /* COULOMB ELECTROSTATICS */
 468             velec            = _mm_mul_ps(qq22,rinv22);
 469             felec            = _mm_mul_ps(velec,rinvsq22);
 470
 471             /* Update potential sum for this i atom from the interaction with this j atom. */
 472             velecsum         = _mm_add_ps(velecsum,velec);
 473
 474             fscal            = felec;
 475
 476              /* Update vectorial force */
 477             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 478             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 479             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 480
 481             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 482             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 483             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 484
 485             fjptrA             = f+j_coord_offsetA;
 486             fjptrB             = f+j_coord_offsetB;
 487             fjptrC             = f+j_coord_offsetC;
 488             fjptrD             = f+j_coord_offsetD;
 489
 490             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 491                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 492
 493             /* Inner loop uses 291 flops */
 494         }
 495
 496         if(jidx<j_index_end)
 497         {
 498
 499             /* Get j neighbor index, and coordinate index */
 500             jnrlistA         = jjnr[jidx];
 501             jnrlistB         = jjnr[jidx+1];
 502             jnrlistC         = jjnr[jidx+2];
 503             jnrlistD         = jjnr[jidx+3];
 504             /* Sign of each element will be negative for non-real atoms.
 505              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 506              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 507              */
 508             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 509             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 510             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 511             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 512             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 513             j_coord_offsetA  = DIM*jnrA;
 514             j_coord_offsetB  = DIM*jnrB;
 515             j_coord_offsetC  = DIM*jnrC;
 516             j_coord_offsetD  = DIM*jnrD;
 517
 518             /* load j atom coordinates */
 519             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 520                                               x+j_coord_offsetC,x+j_coord_offsetD,
 521                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 522
 523             /* Calculate displacement vector */
 524             dx00             = _mm_sub_ps(ix0,jx0);
 525             dy00             = _mm_sub_ps(iy0,jy0);
 526             dz00             = _mm_sub_ps(iz0,jz0);
 527             dx01             = _mm_sub_ps(ix0,jx1);
 528             dy01             = _mm_sub_ps(iy0,jy1);
 529             dz01             = _mm_sub_ps(iz0,jz1);
 530             dx02             = _mm_sub_ps(ix0,jx2);
 531             dy02             = _mm_sub_ps(iy0,jy2);
 532             dz02             = _mm_sub_ps(iz0,jz2);
 533             dx10             = _mm_sub_ps(ix1,jx0);
 534             dy10             = _mm_sub_ps(iy1,jy0);
 535             dz10             = _mm_sub_ps(iz1,jz0);
 536             dx11             = _mm_sub_ps(ix1,jx1);
 537             dy11             = _mm_sub_ps(iy1,jy1);
 538             dz11             = _mm_sub_ps(iz1,jz1);
 539             dx12             = _mm_sub_ps(ix1,jx2);
 540             dy12             = _mm_sub_ps(iy1,jy2);
 541             dz12             = _mm_sub_ps(iz1,jz2);
 542             dx20             = _mm_sub_ps(ix2,jx0);
 543             dy20             = _mm_sub_ps(iy2,jy0);
 544             dz20             = _mm_sub_ps(iz2,jz0);
 545             dx21             = _mm_sub_ps(ix2,jx1);
 546             dy21             = _mm_sub_ps(iy2,jy1);
 547             dz21             = _mm_sub_ps(iz2,jz1);
 548             dx22             = _mm_sub_ps(ix2,jx2);
 549             dy22             = _mm_sub_ps(iy2,jy2);
 550             dz22             = _mm_sub_ps(iz2,jz2);
 551
 552             /* Calculate squared distance and things based on it */
 553             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 554             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 555             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 556             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 557             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 558             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 559             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 560             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 561             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 562
 563             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 564             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 565             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 566             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 567             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 568             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 569             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 570             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 571             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 572
 573             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 574             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
 575             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
 576             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 577             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 578             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 579             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 580             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 581             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 582
 583             fjx0             = _mm_setzero_ps();
 584             fjy0             = _mm_setzero_ps();
 585             fjz0             = _mm_setzero_ps();
 586             fjx1             = _mm_setzero_ps();
 587             fjy1             = _mm_setzero_ps();
 588             fjz1             = _mm_setzero_ps();
 589             fjx2             = _mm_setzero_ps();
 590             fjy2             = _mm_setzero_ps();
 591             fjz2             = _mm_setzero_ps();
 592
 593             /**************************
 594              * CALCULATE INTERACTIONS *
 595              **************************/
 596
 597             /* COULOMB ELECTROSTATICS */
 598             velec            = _mm_mul_ps(qq00,rinv00);
 599             felec            = _mm_mul_ps(velec,rinvsq00);
 600
 601             /* LENNARD-JONES DISPERSION/REPULSION */
 602
 603             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 604             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 605             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 606             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 607             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 608
 609             /* Update potential sum for this i atom from the interaction with this j atom. */
 610             velec            = _mm_andnot_ps(dummy_mask,velec);
 611             velecsum         = _mm_add_ps(velecsum,velec);
 612             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 613             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 614
 615             fscal            = _mm_add_ps(felec,fvdw);
 616
 617             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 618
 619              /* Update vectorial force */
 620             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 621             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 622             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 623
 624             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 625             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 626             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 627
 628             /**************************
 629              * CALCULATE INTERACTIONS *
 630              **************************/
 631
 632             /* COULOMB ELECTROSTATICS */
 633             velec            = _mm_mul_ps(qq01,rinv01);
 634             felec            = _mm_mul_ps(velec,rinvsq01);
 635
 636             /* Update potential sum for this i atom from the interaction with this j atom. */
 637             velec            = _mm_andnot_ps(dummy_mask,velec);
 638             velecsum         = _mm_add_ps(velecsum,velec);
 639
 640             fscal            = felec;
 641
 642             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 643
 644              /* Update vectorial force */
 645             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 646             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 647             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 648
 649             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 650             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 651             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 652
 653             /**************************
 654              * CALCULATE INTERACTIONS *
 655              **************************/
 656
 657             /* COULOMB ELECTROSTATICS */
 658             velec            = _mm_mul_ps(qq02,rinv02);
 659             felec            = _mm_mul_ps(velec,rinvsq02);
 660
 661             /* Update potential sum for this i atom from the interaction with this j atom. */
 662             velec            = _mm_andnot_ps(dummy_mask,velec);
 663             velecsum         = _mm_add_ps(velecsum,velec);
 664
 665             fscal            = felec;
 666
 667             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 668
 669              /* Update vectorial force */
 670             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 671             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 672             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 673
 674             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 675             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 676             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 677
 678             /**************************
 679              * CALCULATE INTERACTIONS *
 680              **************************/
 681
 682             /* COULOMB ELECTROSTATICS */
 683             velec            = _mm_mul_ps(qq10,rinv10);
 684             felec            = _mm_mul_ps(velec,rinvsq10);
 685
 686             /* Update potential sum for this i atom from the interaction with this j atom. */
 687             velec            = _mm_andnot_ps(dummy_mask,velec);
 688             velecsum         = _mm_add_ps(velecsum,velec);
 689
 690             fscal            = felec;
 691
 692             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 693
 694              /* Update vectorial force */
 695             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 696             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 697             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 698
 699             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 700             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 701             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 702
 703             /**************************
 704              * CALCULATE INTERACTIONS *
 705              **************************/
 706
 707             /* COULOMB ELECTROSTATICS */
 708             velec            = _mm_mul_ps(qq11,rinv11);
 709             felec            = _mm_mul_ps(velec,rinvsq11);
 710
 711             /* Update potential sum for this i atom from the interaction with this j atom. */
 712             velec            = _mm_andnot_ps(dummy_mask,velec);
 713             velecsum         = _mm_add_ps(velecsum,velec);
 714
 715             fscal            = felec;
 716
 717             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 718
 719              /* Update vectorial force */
 720             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 721             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 722             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 723
 724             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 725             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 726             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 727
 728             /**************************
 729              * CALCULATE INTERACTIONS *
 730              **************************/
 731
 732             /* COULOMB ELECTROSTATICS */
 733             velec            = _mm_mul_ps(qq12,rinv12);
 734             felec            = _mm_mul_ps(velec,rinvsq12);
 735
 736             /* Update potential sum for this i atom from the interaction with this j atom. */
 737             velec            = _mm_andnot_ps(dummy_mask,velec);
 738             velecsum         = _mm_add_ps(velecsum,velec);
 739
 740             fscal            = felec;
 741
 742             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 743
 744              /* Update vectorial force */
 745             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 746             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 747             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 748
 749             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 750             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 751             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 752
 753             /**************************
 754              * CALCULATE INTERACTIONS *
 755              **************************/
 756
 757             /* COULOMB ELECTROSTATICS */
 758             velec            = _mm_mul_ps(qq20,rinv20);
 759             felec            = _mm_mul_ps(velec,rinvsq20);
 760
 761             /* Update potential sum for this i atom from the interaction with this j atom. */
 762             velec            = _mm_andnot_ps(dummy_mask,velec);
 763             velecsum         = _mm_add_ps(velecsum,velec);
 764
 765             fscal            = felec;
 766
 767             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 768
 769              /* Update vectorial force */
 770             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 771             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 772             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 773
 774             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 775             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 776             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 777
 778             /**************************
 779              * CALCULATE INTERACTIONS *
 780              **************************/
 781
 782             /* COULOMB ELECTROSTATICS */
 783             velec            = _mm_mul_ps(qq21,rinv21);
 784             felec            = _mm_mul_ps(velec,rinvsq21);
 785
 786             /* Update potential sum for this i atom from the interaction with this j atom. */
 787             velec            = _mm_andnot_ps(dummy_mask,velec);
 788             velecsum         = _mm_add_ps(velecsum,velec);
 789
 790             fscal            = felec;
 791
 792             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 793
 794              /* Update vectorial force */
 795             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 796             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 797             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 798
 799             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 800             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 801             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 802
 803             /**************************
 804              * CALCULATE INTERACTIONS *
 805              **************************/
 806
 807             /* COULOMB ELECTROSTATICS */
 808             velec            = _mm_mul_ps(qq22,rinv22);
 809             felec            = _mm_mul_ps(velec,rinvsq22);
 810
 811             /* Update potential sum for this i atom from the interaction with this j atom. */
 812             velec            = _mm_andnot_ps(dummy_mask,velec);
 813             velecsum         = _mm_add_ps(velecsum,velec);
 814
 815             fscal            = felec;
 816
 817             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 818
 819              /* Update vectorial force */
 820             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 821             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 822             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 823
 824             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 825             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 826             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 827
 828             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
 829             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
 830             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
 831             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
 832
 833             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 834                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 835
 836             /* Inner loop uses 291 flops */
 837         }
 838
 839         /* End of innermost loop */
 840
 841         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
 842                                               f+i_coord_offset,fshift+i_shift_offset);
 843
 844         ggid                        = gid[iidx];
 845         /* Update potential energies */
 846         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
 847         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
 848
 849         /* Increment number of inner iterations */
 850         inneriter                  += j_index_end - j_index_start;
 851
 852         /* Outer loop uses 20 flops */
 853     }
 854
 855     /* Increment number of outer iterations */
 856     outeriter        += nri;
 857
 858     /* Update outer/inner flops */
 859
 860     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*291);
 861 }
 862 /*
 863  * Gromacs nonbonded kernel:   nb_kernel_ElecCoul_VdwLJ_GeomW3W3_F_avx_128_fma_single
 864  * Electrostatics interaction: Coulomb
 865  * VdW interaction:            LennardJones
 866  * Geometry:                   Water3-Water3
 867  * Calculate force/pot:        Force
 868  */
 869 void
 870 nb_kernel_ElecCoul_VdwLJ_GeomW3W3_F_avx_128_fma_single
 871                     (t_nblist * gmx_restrict                nlist,
 872                      rvec * gmx_restrict                    xx,
 873                      rvec * gmx_restrict                    ff,
 874                      t_forcerec * gmx_restrict              fr,
 875                      t_mdatoms * gmx_restrict               mdatoms,
 876                      nb_kernel_data_t * gmx_restrict        kernel_data,
 877                      t_nrnb * gmx_restrict                  nrnb)
 878 {
 879     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 880      * just 0 for non-waters.
 881      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
 882      * jnr indices corresponding to data put in the four positions in the SIMD register.
 883      */
 884     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 885     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 886     int              jnrA,jnrB,jnrC,jnrD;
 887     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
 888     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
 889     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 890     real             rcutoff_scalar;
 891     real             *shiftvec,*fshift,*x,*f;
 892     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
 893     real             scratch[4*DIM];
 894     __m128           fscal,rcutoff,rcutoff2,jidxall;
 895     int              vdwioffset0;
 896     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 897     int              vdwioffset1;
 898     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 899     int              vdwioffset2;
 900     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 901     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
 902     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 903     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
 904     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
 905     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
 906     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
 907     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 908     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
 909     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
 910     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 911     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 912     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 913     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 914     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 915     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 916     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 917     real             *charge;
 918     int              nvdwtype;
 919     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 920     int              *vdwtype;
 921     real             *vdwparam;
 922     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 923     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 924     __m128           dummy_mask,cutoff_mask;
 925     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 926     __m128           one     = _mm_set1_ps(1.0);
 927     __m128           two     = _mm_set1_ps(2.0);
 928     x                = xx[0];
 929     f                = ff[0];
 930
 931     nri              = nlist->nri;
 932     iinr             = nlist->iinr;
 933     jindex           = nlist->jindex;
 934     jjnr             = nlist->jjnr;
 935     shiftidx         = nlist->shift;
 936     gid              = nlist->gid;
 937     shiftvec         = fr->shift_vec[0];
 938     fshift           = fr->fshift[0];
 939     facel            = _mm_set1_ps(fr->epsfac);
 940     charge           = mdatoms->chargeA;
 941     nvdwtype         = fr->ntype;
 942     vdwparam         = fr->nbfp;
 943     vdwtype          = mdatoms->typeA;
 944
 945     /* Setup water-specific parameters */
 946     inr              = nlist->iinr[0];
 947     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 948     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 949     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 950     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 951
 952     jq0              = _mm_set1_ps(charge[inr+0]);
 953     jq1              = _mm_set1_ps(charge[inr+1]);
 954     jq2              = _mm_set1_ps(charge[inr+2]);
 955     vdwjidx0A        = 2*vdwtype[inr+0];
 956     qq00             = _mm_mul_ps(iq0,jq0);
 957     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 958     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 959     qq01             = _mm_mul_ps(iq0,jq1);
 960     qq02             = _mm_mul_ps(iq0,jq2);
 961     qq10             = _mm_mul_ps(iq1,jq0);
 962     qq11             = _mm_mul_ps(iq1,jq1);
 963     qq12             = _mm_mul_ps(iq1,jq2);
 964     qq20             = _mm_mul_ps(iq2,jq0);
 965     qq21             = _mm_mul_ps(iq2,jq1);
 966     qq22             = _mm_mul_ps(iq2,jq2);
 967
 968     /* Avoid stupid compiler warnings */
 969     jnrA = jnrB = jnrC = jnrD = 0;
 970     j_coord_offsetA = 0;
 971     j_coord_offsetB = 0;
 972     j_coord_offsetC = 0;
 973     j_coord_offsetD = 0;
 974
 975     outeriter        = 0;
 976     inneriter        = 0;
 977
 978     for(iidx=0;iidx<4*DIM;iidx++)
 979     {
 980         scratch[iidx] = 0.0;
 981     }
 982
 983     /* Start outer loop over neighborlists */
 984     for(iidx=0; iidx<nri; iidx++)
 985     {
 986         /* Load shift vector for this list */
 987         i_shift_offset   = DIM*shiftidx[iidx];
 988
 989         /* Load limits for loop over neighbors */
 990         j_index_start    = jindex[iidx];
 991         j_index_end      = jindex[iidx+1];
 992
 993         /* Get outer coordinate index */
 994         inr              = iinr[iidx];
 995         i_coord_offset   = DIM*inr;
 996
 997         /* Load i particle coords and add shift vector */
 998         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 999                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1000
1001         fix0             = _mm_setzero_ps();
1002         fiy0             = _mm_setzero_ps();
1003         fiz0             = _mm_setzero_ps();
1004         fix1             = _mm_setzero_ps();
1005         fiy1             = _mm_setzero_ps();
1006         fiz1             = _mm_setzero_ps();
1007         fix2             = _mm_setzero_ps();
1008         fiy2             = _mm_setzero_ps();
1009         fiz2             = _mm_setzero_ps();
1010
1011         /* Start inner kernel loop */
1012         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1013         {
1014
1015             /* Get j neighbor index, and coordinate index */
1016             jnrA             = jjnr[jidx];
1017             jnrB             = jjnr[jidx+1];
1018             jnrC             = jjnr[jidx+2];
1019             jnrD             = jjnr[jidx+3];
1020             j_coord_offsetA  = DIM*jnrA;
1021             j_coord_offsetB  = DIM*jnrB;
1022             j_coord_offsetC  = DIM*jnrC;
1023             j_coord_offsetD  = DIM*jnrD;
1024
1025             /* load j atom coordinates */
1026             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1027                                               x+j_coord_offsetC,x+j_coord_offsetD,
1028                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1029
1030             /* Calculate displacement vector */
1031             dx00             = _mm_sub_ps(ix0,jx0);
1032             dy00             = _mm_sub_ps(iy0,jy0);
1033             dz00             = _mm_sub_ps(iz0,jz0);
1034             dx01             = _mm_sub_ps(ix0,jx1);
1035             dy01             = _mm_sub_ps(iy0,jy1);
1036             dz01             = _mm_sub_ps(iz0,jz1);
1037             dx02             = _mm_sub_ps(ix0,jx2);
1038             dy02             = _mm_sub_ps(iy0,jy2);
1039             dz02             = _mm_sub_ps(iz0,jz2);
1040             dx10             = _mm_sub_ps(ix1,jx0);
1041             dy10             = _mm_sub_ps(iy1,jy0);
1042             dz10             = _mm_sub_ps(iz1,jz0);
1043             dx11             = _mm_sub_ps(ix1,jx1);
1044             dy11             = _mm_sub_ps(iy1,jy1);
1045             dz11             = _mm_sub_ps(iz1,jz1);
1046             dx12             = _mm_sub_ps(ix1,jx2);
1047             dy12             = _mm_sub_ps(iy1,jy2);
1048             dz12             = _mm_sub_ps(iz1,jz2);
1049             dx20             = _mm_sub_ps(ix2,jx0);
1050             dy20             = _mm_sub_ps(iy2,jy0);
1051             dz20             = _mm_sub_ps(iz2,jz0);
1052             dx21             = _mm_sub_ps(ix2,jx1);
1053             dy21             = _mm_sub_ps(iy2,jy1);
1054             dz21             = _mm_sub_ps(iz2,jz1);
1055             dx22             = _mm_sub_ps(ix2,jx2);
1056             dy22             = _mm_sub_ps(iy2,jy2);
1057             dz22             = _mm_sub_ps(iz2,jz2);
1058
1059             /* Calculate squared distance and things based on it */
1060             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1061             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1062             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1063             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1064             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1065             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1066             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1067             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1068             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1069
1070             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1071             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1072             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1073             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1074             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1075             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1076             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1077             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1078             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1079
1080             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1081             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
1082             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
1083             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1084             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1085             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1086             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1087             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1088             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1089
1090             fjx0             = _mm_setzero_ps();
1091             fjy0             = _mm_setzero_ps();
1092             fjz0             = _mm_setzero_ps();
1093             fjx1             = _mm_setzero_ps();
1094             fjy1             = _mm_setzero_ps();
1095             fjz1             = _mm_setzero_ps();
1096             fjx2             = _mm_setzero_ps();
1097             fjy2             = _mm_setzero_ps();
1098             fjz2             = _mm_setzero_ps();
1099
1100             /**************************
1101              * CALCULATE INTERACTIONS *
1102              **************************/
1103
1104             /* COULOMB ELECTROSTATICS */
1105             velec            = _mm_mul_ps(qq00,rinv00);
1106             felec            = _mm_mul_ps(velec,rinvsq00);
1107
1108             /* LENNARD-JONES DISPERSION/REPULSION */
1109
1110             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1111             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1112
1113             fscal            = _mm_add_ps(felec,fvdw);
1114
1115              /* Update vectorial force */
1116             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1117             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1118             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1119
1120             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1121             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1122             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1123
1124             /**************************
1125              * CALCULATE INTERACTIONS *
1126              **************************/
1127
1128             /* COULOMB ELECTROSTATICS */
1129             velec            = _mm_mul_ps(qq01,rinv01);
1130             felec            = _mm_mul_ps(velec,rinvsq01);
1131
1132             fscal            = felec;
1133
1134              /* Update vectorial force */
1135             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1136             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1137             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1138
1139             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1140             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1141             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1142
1143             /**************************
1144              * CALCULATE INTERACTIONS *
1145              **************************/
1146
1147             /* COULOMB ELECTROSTATICS */
1148             velec            = _mm_mul_ps(qq02,rinv02);
1149             felec            = _mm_mul_ps(velec,rinvsq02);
1150
1151             fscal            = felec;
1152
1153              /* Update vectorial force */
1154             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1155             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1156             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1157
1158             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1159             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1160             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1161
1162             /**************************
1163              * CALCULATE INTERACTIONS *
1164              **************************/
1165
1166             /* COULOMB ELECTROSTATICS */
1167             velec            = _mm_mul_ps(qq10,rinv10);
1168             felec            = _mm_mul_ps(velec,rinvsq10);
1169
1170             fscal            = felec;
1171
1172              /* Update vectorial force */
1173             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1174             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1175             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1176
1177             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1178             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1179             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1180
1181             /**************************
1182              * CALCULATE INTERACTIONS *
1183              **************************/
1184
1185             /* COULOMB ELECTROSTATICS */
1186             velec            = _mm_mul_ps(qq11,rinv11);
1187             felec            = _mm_mul_ps(velec,rinvsq11);
1188
1189             fscal            = felec;
1190
1191              /* Update vectorial force */
1192             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1193             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1194             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1195
1196             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1197             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1198             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1199
1200             /**************************
1201              * CALCULATE INTERACTIONS *
1202              **************************/
1203
1204             /* COULOMB ELECTROSTATICS */
1205             velec            = _mm_mul_ps(qq12,rinv12);
1206             felec            = _mm_mul_ps(velec,rinvsq12);
1207
1208             fscal            = felec;
1209
1210              /* Update vectorial force */
1211             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1212             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1213             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1214
1215             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1216             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1217             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1218
1219             /**************************
1220              * CALCULATE INTERACTIONS *
1221              **************************/
1222
1223             /* COULOMB ELECTROSTATICS */
1224             velec            = _mm_mul_ps(qq20,rinv20);
1225             felec            = _mm_mul_ps(velec,rinvsq20);
1226
1227             fscal            = felec;
1228
1229              /* Update vectorial force */
1230             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1231             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1232             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1233
1234             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1235             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1236             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1237
1238             /**************************
1239              * CALCULATE INTERACTIONS *
1240              **************************/
1241
1242             /* COULOMB ELECTROSTATICS */
1243             velec            = _mm_mul_ps(qq21,rinv21);
1244             felec            = _mm_mul_ps(velec,rinvsq21);
1245
1246             fscal            = felec;
1247
1248              /* Update vectorial force */
1249             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1250             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1251             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1252
1253             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1254             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1255             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1256
1257             /**************************
1258              * CALCULATE INTERACTIONS *
1259              **************************/
1260
1261             /* COULOMB ELECTROSTATICS */
1262             velec            = _mm_mul_ps(qq22,rinv22);
1263             felec            = _mm_mul_ps(velec,rinvsq22);
1264
1265             fscal            = felec;
1266
1267              /* Update vectorial force */
1268             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1269             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1270             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1271
1272             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1273             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1274             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1275
1276             fjptrA             = f+j_coord_offsetA;
1277             fjptrB             = f+j_coord_offsetB;
1278             fjptrC             = f+j_coord_offsetC;
1279             fjptrD             = f+j_coord_offsetD;
1280
1281             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1282                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1283
1284             /* Inner loop uses 277 flops */
1285         }
1286
1287         if(jidx<j_index_end)
1288         {
1289
1290             /* Get j neighbor index, and coordinate index */
1291             jnrlistA         = jjnr[jidx];
1292             jnrlistB         = jjnr[jidx+1];
1293             jnrlistC         = jjnr[jidx+2];
1294             jnrlistD         = jjnr[jidx+3];
1295             /* Sign of each element will be negative for non-real atoms.
1296              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1297              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1298              */
1299             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1300             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
1301             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
1302             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1303             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1304             j_coord_offsetA  = DIM*jnrA;
1305             j_coord_offsetB  = DIM*jnrB;
1306             j_coord_offsetC  = DIM*jnrC;
1307             j_coord_offsetD  = DIM*jnrD;
1308
1309             /* load j atom coordinates */
1310             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1311                                               x+j_coord_offsetC,x+j_coord_offsetD,
1312                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1313
1314             /* Calculate displacement vector */
1315             dx00             = _mm_sub_ps(ix0,jx0);
1316             dy00             = _mm_sub_ps(iy0,jy0);
1317             dz00             = _mm_sub_ps(iz0,jz0);
1318             dx01             = _mm_sub_ps(ix0,jx1);
1319             dy01             = _mm_sub_ps(iy0,jy1);
1320             dz01             = _mm_sub_ps(iz0,jz1);
1321             dx02             = _mm_sub_ps(ix0,jx2);
1322             dy02             = _mm_sub_ps(iy0,jy2);
1323             dz02             = _mm_sub_ps(iz0,jz2);
1324             dx10             = _mm_sub_ps(ix1,jx0);
1325             dy10             = _mm_sub_ps(iy1,jy0);
1326             dz10             = _mm_sub_ps(iz1,jz0);
1327             dx11             = _mm_sub_ps(ix1,jx1);
1328             dy11             = _mm_sub_ps(iy1,jy1);
1329             dz11             = _mm_sub_ps(iz1,jz1);
1330             dx12             = _mm_sub_ps(ix1,jx2);
1331             dy12             = _mm_sub_ps(iy1,jy2);
1332             dz12             = _mm_sub_ps(iz1,jz2);
1333             dx20             = _mm_sub_ps(ix2,jx0);
1334             dy20             = _mm_sub_ps(iy2,jy0);
1335             dz20             = _mm_sub_ps(iz2,jz0);
1336             dx21             = _mm_sub_ps(ix2,jx1);
1337             dy21             = _mm_sub_ps(iy2,jy1);
1338             dz21             = _mm_sub_ps(iz2,jz1);
1339             dx22             = _mm_sub_ps(ix2,jx2);
1340             dy22             = _mm_sub_ps(iy2,jy2);
1341             dz22             = _mm_sub_ps(iz2,jz2);
1342
1343             /* Calculate squared distance and things based on it */
1344             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1345             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1346             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1347             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1348             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1349             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1350             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1351             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1352             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1353
1354             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1355             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1356             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1357             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1358             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1359             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1360             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1361             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1362             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1363
1364             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1365             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
1366             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
1367             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1368             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1369             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1370             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1371             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1372             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1373
1374             fjx0             = _mm_setzero_ps();
1375             fjy0             = _mm_setzero_ps();
1376             fjz0             = _mm_setzero_ps();
1377             fjx1             = _mm_setzero_ps();
1378             fjy1             = _mm_setzero_ps();
1379             fjz1             = _mm_setzero_ps();
1380             fjx2             = _mm_setzero_ps();
1381             fjy2             = _mm_setzero_ps();
1382             fjz2             = _mm_setzero_ps();
1383
1384             /**************************
1385              * CALCULATE INTERACTIONS *
1386              **************************/
1387
1388             /* COULOMB ELECTROSTATICS */
1389             velec            = _mm_mul_ps(qq00,rinv00);
1390             felec            = _mm_mul_ps(velec,rinvsq00);
1391
1392             /* LENNARD-JONES DISPERSION/REPULSION */
1393
1394             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1395             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1396
1397             fscal            = _mm_add_ps(felec,fvdw);
1398
1399             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1400
1401              /* Update vectorial force */
1402             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1403             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1404             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1405
1406             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1407             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1408             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1409
1410             /**************************
1411              * CALCULATE INTERACTIONS *
1412              **************************/
1413
1414             /* COULOMB ELECTROSTATICS */
1415             velec            = _mm_mul_ps(qq01,rinv01);
1416             felec            = _mm_mul_ps(velec,rinvsq01);
1417
1418             fscal            = felec;
1419
1420             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1421
1422              /* Update vectorial force */
1423             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1424             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1425             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1426
1427             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1428             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1429             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1430
1431             /**************************
1432              * CALCULATE INTERACTIONS *
1433              **************************/
1434
1435             /* COULOMB ELECTROSTATICS */
1436             velec            = _mm_mul_ps(qq02,rinv02);
1437             felec            = _mm_mul_ps(velec,rinvsq02);
1438
1439             fscal            = felec;
1440
1441             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1442
1443              /* Update vectorial force */
1444             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1445             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1446             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1447
1448             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1449             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1450             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1451
1452             /**************************
1453              * CALCULATE INTERACTIONS *
1454              **************************/
1455
1456             /* COULOMB ELECTROSTATICS */
1457             velec            = _mm_mul_ps(qq10,rinv10);
1458             felec            = _mm_mul_ps(velec,rinvsq10);
1459
1460             fscal            = felec;
1461
1462             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1463
1464              /* Update vectorial force */
1465             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1466             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1467             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1468
1469             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1470             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1471             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1472
1473             /**************************
1474              * CALCULATE INTERACTIONS *
1475              **************************/
1476
1477             /* COULOMB ELECTROSTATICS */
1478             velec            = _mm_mul_ps(qq11,rinv11);
1479             felec            = _mm_mul_ps(velec,rinvsq11);
1480
1481             fscal            = felec;
1482
1483             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1484
1485              /* Update vectorial force */
1486             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1487             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1488             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1489
1490             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1491             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1492             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1493
1494             /**************************
1495              * CALCULATE INTERACTIONS *
1496              **************************/
1497
1498             /* COULOMB ELECTROSTATICS */
1499             velec            = _mm_mul_ps(qq12,rinv12);
1500             felec            = _mm_mul_ps(velec,rinvsq12);
1501
1502             fscal            = felec;
1503
1504             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1505
1506              /* Update vectorial force */
1507             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1508             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1509             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1510
1511             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1512             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1513             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1514
1515             /**************************
1516              * CALCULATE INTERACTIONS *
1517              **************************/
1518
1519             /* COULOMB ELECTROSTATICS */
1520             velec            = _mm_mul_ps(qq20,rinv20);
1521             felec            = _mm_mul_ps(velec,rinvsq20);
1522
1523             fscal            = felec;
1524
1525             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1526
1527              /* Update vectorial force */
1528             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1529             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1530             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1531
1532             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1533             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1534             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1535
1536             /**************************
1537              * CALCULATE INTERACTIONS *
1538              **************************/
1539
1540             /* COULOMB ELECTROSTATICS */
1541             velec            = _mm_mul_ps(qq21,rinv21);
1542             felec            = _mm_mul_ps(velec,rinvsq21);
1543
1544             fscal            = felec;
1545
1546             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1547
1548              /* Update vectorial force */
1549             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1550             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1551             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1552
1553             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1554             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1555             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1556
1557             /**************************
1558              * CALCULATE INTERACTIONS *
1559              **************************/
1560
1561             /* COULOMB ELECTROSTATICS */
1562             velec            = _mm_mul_ps(qq22,rinv22);
1563             felec            = _mm_mul_ps(velec,rinvsq22);
1564
1565             fscal            = felec;
1566
1567             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1568
1569              /* Update vectorial force */
1570             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1571             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1572             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1573
1574             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1575             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1576             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1577
1578             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1579             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1580             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1581             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1582
1583             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1584                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1585
1586             /* Inner loop uses 277 flops */
1587         }
1588
1589         /* End of innermost loop */
1590
1591         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1592                                               f+i_coord_offset,fshift+i_shift_offset);
1593
1594         /* Increment number of inner iterations */
1595         inneriter                  += j_index_end - j_index_start;
1596
1597         /* Outer loop uses 18 flops */
1598     }
1599
1600     /* Increment number of outer iterations */
1601     outeriter        += nri;
1602
1603     /* Update outer/inner flops */
1604
1605     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*277);
1606 }