src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecEw_VdwLJ_GeomW4P1_avx_128_fma_single.c

   1 /*
   2  * Note: this file was generated by the Gromacs avx_128_fma_single kernel generator.
   3  *
   4  *                This source code is part of
   5  *
   6  *                 G   R   O   M   A   C   S
   7  *
   8  * Copyright (c) 2001-2012, The GROMACS Development Team
   9  *
  10  * Gromacs is a library for molecular simulation and trajectory analysis,
  11  * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
  12  * a full list of developers and information, check out http://www.gromacs.org
  13  *
  14  * This program is free software; you can redistribute it and/or modify it under
  15  * the terms of the GNU Lesser General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option) any
  17  * later version.
  18  *
  19  * To help fund GROMACS development, we humbly ask that you cite
  20  * the papers people have written on it - you can find them on the website.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include <config.h>
  24 #endif
  25
  26 #include <math.h>
  27
  28 #include "../nb_kernel.h"
  29 #include "types/simple.h"
  30 #include "vec.h"
  31 #include "nrnb.h"
  32
  33 #include "gmx_math_x86_avx_128_fma_single.h"
  34 #include "kernelutil_x86_avx_128_fma_single.h"
  35
  36 /*
  37  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4P1_VF_avx_128_fma_single
  38  * Electrostatics interaction: Ewald
  39  * VdW interaction:            LennardJones
  40  * Geometry:                   Water4-Particle
  41  * Calculate force/pot:        PotentialAndForce
  42  */
  43 void
  44 nb_kernel_ElecEw_VdwLJ_GeomW4P1_VF_avx_128_fma_single
  45                     (t_nblist * gmx_restrict                nlist,
  46                      rvec * gmx_restrict                    xx,
  47                      rvec * gmx_restrict                    ff,
  48                      t_forcerec * gmx_restrict              fr,
  49                      t_mdatoms * gmx_restrict               mdatoms,
  50                      nb_kernel_data_t * gmx_restrict        kernel_data,
  51                      t_nrnb * gmx_restrict                  nrnb)
  52 {
  53     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  54      * just 0 for non-waters.
  55      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  56      * jnr indices corresponding to data put in the four positions in the SIMD register.
  57      */
  58     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  59     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  60     int              jnrA,jnrB,jnrC,jnrD;
  61     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  62     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  63     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  64     real             rcutoff_scalar;
  65     real             *shiftvec,*fshift,*x,*f;
  66     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  67     real             scratch[4*DIM];
  68     __m128           fscal,rcutoff,rcutoff2,jidxall;
  69     int              vdwioffset0;
  70     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  71     int              vdwioffset1;
  72     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  73     int              vdwioffset2;
  74     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  75     int              vdwioffset3;
  76     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
  77     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  78     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  79     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  80     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  81     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
  82     __m128           dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
  83     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
  84     real             *charge;
  85     int              nvdwtype;
  86     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
  87     int              *vdwtype;
  88     real             *vdwparam;
  89     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
  90     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
  91     __m128i          ewitab;
  92     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
  93     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
  94     real             *ewtab;
  95     __m128           dummy_mask,cutoff_mask;
  96     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
  97     __m128           one     = _mm_set1_ps(1.0);
  98     __m128           two     = _mm_set1_ps(2.0);
  99     x                = xx[0];
 100     f                = ff[0];
 101
 102     nri              = nlist->nri;
 103     iinr             = nlist->iinr;
 104     jindex           = nlist->jindex;
 105     jjnr             = nlist->jjnr;
 106     shiftidx         = nlist->shift;
 107     gid              = nlist->gid;
 108     shiftvec         = fr->shift_vec[0];
 109     fshift           = fr->fshift[0];
 110     facel            = _mm_set1_ps(fr->epsfac);
 111     charge           = mdatoms->chargeA;
 112     nvdwtype         = fr->ntype;
 113     vdwparam         = fr->nbfp;
 114     vdwtype          = mdatoms->typeA;
 115
 116     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
 117     beta             = _mm_set1_ps(fr->ic->ewaldcoeff);
 118     beta2            = _mm_mul_ps(beta,beta);
 119     beta3            = _mm_mul_ps(beta,beta2);
 120     ewtab            = fr->ic->tabq_coul_FDV0;
 121     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
 122     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
 123
 124     /* Setup water-specific parameters */
 125     inr              = nlist->iinr[0];
 126     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 127     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 128     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
 129     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 130
 131     /* Avoid stupid compiler warnings */
 132     jnrA = jnrB = jnrC = jnrD = 0;
 133     j_coord_offsetA = 0;
 134     j_coord_offsetB = 0;
 135     j_coord_offsetC = 0;
 136     j_coord_offsetD = 0;
 137
 138     outeriter        = 0;
 139     inneriter        = 0;
 140
 141     for(iidx=0;iidx<4*DIM;iidx++)
 142     {
 143         scratch[iidx] = 0.0;
 144     }
 145
 146     /* Start outer loop over neighborlists */
 147     for(iidx=0; iidx<nri; iidx++)
 148     {
 149         /* Load shift vector for this list */
 150         i_shift_offset   = DIM*shiftidx[iidx];
 151
 152         /* Load limits for loop over neighbors */
 153         j_index_start    = jindex[iidx];
 154         j_index_end      = jindex[iidx+1];
 155
 156         /* Get outer coordinate index */
 157         inr              = iinr[iidx];
 158         i_coord_offset   = DIM*inr;
 159
 160         /* Load i particle coords and add shift vector */
 161         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 162                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 163
 164         fix0             = _mm_setzero_ps();
 165         fiy0             = _mm_setzero_ps();
 166         fiz0             = _mm_setzero_ps();
 167         fix1             = _mm_setzero_ps();
 168         fiy1             = _mm_setzero_ps();
 169         fiz1             = _mm_setzero_ps();
 170         fix2             = _mm_setzero_ps();
 171         fiy2             = _mm_setzero_ps();
 172         fiz2             = _mm_setzero_ps();
 173         fix3             = _mm_setzero_ps();
 174         fiy3             = _mm_setzero_ps();
 175         fiz3             = _mm_setzero_ps();
 176
 177         /* Reset potential sums */
 178         velecsum         = _mm_setzero_ps();
 179         vvdwsum          = _mm_setzero_ps();
 180
 181         /* Start inner kernel loop */
 182         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 183         {
 184
 185             /* Get j neighbor index, and coordinate index */
 186             jnrA             = jjnr[jidx];
 187             jnrB             = jjnr[jidx+1];
 188             jnrC             = jjnr[jidx+2];
 189             jnrD             = jjnr[jidx+3];
 190             j_coord_offsetA  = DIM*jnrA;
 191             j_coord_offsetB  = DIM*jnrB;
 192             j_coord_offsetC  = DIM*jnrC;
 193             j_coord_offsetD  = DIM*jnrD;
 194
 195             /* load j atom coordinates */
 196             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 197                                               x+j_coord_offsetC,x+j_coord_offsetD,
 198                                               &jx0,&jy0,&jz0);
 199
 200             /* Calculate displacement vector */
 201             dx00             = _mm_sub_ps(ix0,jx0);
 202             dy00             = _mm_sub_ps(iy0,jy0);
 203             dz00             = _mm_sub_ps(iz0,jz0);
 204             dx10             = _mm_sub_ps(ix1,jx0);
 205             dy10             = _mm_sub_ps(iy1,jy0);
 206             dz10             = _mm_sub_ps(iz1,jz0);
 207             dx20             = _mm_sub_ps(ix2,jx0);
 208             dy20             = _mm_sub_ps(iy2,jy0);
 209             dz20             = _mm_sub_ps(iz2,jz0);
 210             dx30             = _mm_sub_ps(ix3,jx0);
 211             dy30             = _mm_sub_ps(iy3,jy0);
 212             dz30             = _mm_sub_ps(iz3,jz0);
 213
 214             /* Calculate squared distance and things based on it */
 215             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 216             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 217             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 218             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 219
 220             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 221             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 222             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 223
 224             rinvsq00         = gmx_mm_inv_ps(rsq00);
 225             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 226             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 227             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 228
 229             /* Load parameters for j particles */
 230             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 231                                                               charge+jnrC+0,charge+jnrD+0);
 232             vdwjidx0A        = 2*vdwtype[jnrA+0];
 233             vdwjidx0B        = 2*vdwtype[jnrB+0];
 234             vdwjidx0C        = 2*vdwtype[jnrC+0];
 235             vdwjidx0D        = 2*vdwtype[jnrD+0];
 236
 237             fjx0             = _mm_setzero_ps();
 238             fjy0             = _mm_setzero_ps();
 239             fjz0             = _mm_setzero_ps();
 240
 241             /**************************
 242              * CALCULATE INTERACTIONS *
 243              **************************/
 244
 245             /* Compute parameters for interactions between i and j atoms */
 246             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 247                                          vdwparam+vdwioffset0+vdwjidx0B,
 248                                          vdwparam+vdwioffset0+vdwjidx0C,
 249                                          vdwparam+vdwioffset0+vdwjidx0D,
 250                                          &c6_00,&c12_00);
 251
 252             /* LENNARD-JONES DISPERSION/REPULSION */
 253
 254             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 255             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 256             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 257             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 258             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 259
 260             /* Update potential sum for this i atom from the interaction with this j atom. */
 261             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 262
 263             fscal            = fvdw;
 264
 265              /* Update vectorial force */
 266             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 267             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 268             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 269
 270             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 271             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 272             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 273
 274             /**************************
 275              * CALCULATE INTERACTIONS *
 276              **************************/
 277
 278             r10              = _mm_mul_ps(rsq10,rinv10);
 279
 280             /* Compute parameters for interactions between i and j atoms */
 281             qq10             = _mm_mul_ps(iq1,jq0);
 282
 283             /* EWALD ELECTROSTATICS */
 284
 285             /* Analytical PME correction */
 286             zeta2            = _mm_mul_ps(beta2,rsq10);
 287             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 288             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 289             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 290             felec            = _mm_mul_ps(qq10,felec);
 291             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 292             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv10);
 293             velec            = _mm_mul_ps(qq10,velec);
 294
 295             /* Update potential sum for this i atom from the interaction with this j atom. */
 296             velecsum         = _mm_add_ps(velecsum,velec);
 297
 298             fscal            = felec;
 299
 300              /* Update vectorial force */
 301             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 302             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 303             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 304
 305             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 306             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 307             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 308
 309             /**************************
 310              * CALCULATE INTERACTIONS *
 311              **************************/
 312
 313             r20              = _mm_mul_ps(rsq20,rinv20);
 314
 315             /* Compute parameters for interactions between i and j atoms */
 316             qq20             = _mm_mul_ps(iq2,jq0);
 317
 318             /* EWALD ELECTROSTATICS */
 319
 320             /* Analytical PME correction */
 321             zeta2            = _mm_mul_ps(beta2,rsq20);
 322             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 323             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 324             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 325             felec            = _mm_mul_ps(qq20,felec);
 326             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 327             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv20);
 328             velec            = _mm_mul_ps(qq20,velec);
 329
 330             /* Update potential sum for this i atom from the interaction with this j atom. */
 331             velecsum         = _mm_add_ps(velecsum,velec);
 332
 333             fscal            = felec;
 334
 335              /* Update vectorial force */
 336             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 337             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 338             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 339
 340             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 341             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 342             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 343
 344             /**************************
 345              * CALCULATE INTERACTIONS *
 346              **************************/
 347
 348             r30              = _mm_mul_ps(rsq30,rinv30);
 349
 350             /* Compute parameters for interactions between i and j atoms */
 351             qq30             = _mm_mul_ps(iq3,jq0);
 352
 353             /* EWALD ELECTROSTATICS */
 354
 355             /* Analytical PME correction */
 356             zeta2            = _mm_mul_ps(beta2,rsq30);
 357             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 358             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 359             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 360             felec            = _mm_mul_ps(qq30,felec);
 361             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 362             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv30);
 363             velec            = _mm_mul_ps(qq30,velec);
 364
 365             /* Update potential sum for this i atom from the interaction with this j atom. */
 366             velecsum         = _mm_add_ps(velecsum,velec);
 367
 368             fscal            = felec;
 369
 370              /* Update vectorial force */
 371             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 372             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 373             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 374
 375             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 376             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 377             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 378
 379             fjptrA             = f+j_coord_offsetA;
 380             fjptrB             = f+j_coord_offsetB;
 381             fjptrC             = f+j_coord_offsetC;
 382             fjptrD             = f+j_coord_offsetD;
 383
 384             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 385
 386             /* Inner loop uses 122 flops */
 387         }
 388
 389         if(jidx<j_index_end)
 390         {
 391
 392             /* Get j neighbor index, and coordinate index */
 393             jnrlistA         = jjnr[jidx];
 394             jnrlistB         = jjnr[jidx+1];
 395             jnrlistC         = jjnr[jidx+2];
 396             jnrlistD         = jjnr[jidx+3];
 397             /* Sign of each element will be negative for non-real atoms.
 398              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 399              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 400              */
 401             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 402             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 403             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 404             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 405             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 406             j_coord_offsetA  = DIM*jnrA;
 407             j_coord_offsetB  = DIM*jnrB;
 408             j_coord_offsetC  = DIM*jnrC;
 409             j_coord_offsetD  = DIM*jnrD;
 410
 411             /* load j atom coordinates */
 412             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 413                                               x+j_coord_offsetC,x+j_coord_offsetD,
 414                                               &jx0,&jy0,&jz0);
 415
 416             /* Calculate displacement vector */
 417             dx00             = _mm_sub_ps(ix0,jx0);
 418             dy00             = _mm_sub_ps(iy0,jy0);
 419             dz00             = _mm_sub_ps(iz0,jz0);
 420             dx10             = _mm_sub_ps(ix1,jx0);
 421             dy10             = _mm_sub_ps(iy1,jy0);
 422             dz10             = _mm_sub_ps(iz1,jz0);
 423             dx20             = _mm_sub_ps(ix2,jx0);
 424             dy20             = _mm_sub_ps(iy2,jy0);
 425             dz20             = _mm_sub_ps(iz2,jz0);
 426             dx30             = _mm_sub_ps(ix3,jx0);
 427             dy30             = _mm_sub_ps(iy3,jy0);
 428             dz30             = _mm_sub_ps(iz3,jz0);
 429
 430             /* Calculate squared distance and things based on it */
 431             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 432             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 433             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 434             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 435
 436             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 437             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 438             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 439
 440             rinvsq00         = gmx_mm_inv_ps(rsq00);
 441             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 442             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 443             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 444
 445             /* Load parameters for j particles */
 446             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 447                                                               charge+jnrC+0,charge+jnrD+0);
 448             vdwjidx0A        = 2*vdwtype[jnrA+0];
 449             vdwjidx0B        = 2*vdwtype[jnrB+0];
 450             vdwjidx0C        = 2*vdwtype[jnrC+0];
 451             vdwjidx0D        = 2*vdwtype[jnrD+0];
 452
 453             fjx0             = _mm_setzero_ps();
 454             fjy0             = _mm_setzero_ps();
 455             fjz0             = _mm_setzero_ps();
 456
 457             /**************************
 458              * CALCULATE INTERACTIONS *
 459              **************************/
 460
 461             /* Compute parameters for interactions between i and j atoms */
 462             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 463                                          vdwparam+vdwioffset0+vdwjidx0B,
 464                                          vdwparam+vdwioffset0+vdwjidx0C,
 465                                          vdwparam+vdwioffset0+vdwjidx0D,
 466                                          &c6_00,&c12_00);
 467
 468             /* LENNARD-JONES DISPERSION/REPULSION */
 469
 470             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 471             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 472             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 473             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 474             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 475
 476             /* Update potential sum for this i atom from the interaction with this j atom. */
 477             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 478             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 479
 480             fscal            = fvdw;
 481
 482             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 483
 484              /* Update vectorial force */
 485             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 486             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 487             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 488
 489             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 490             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 491             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 492
 493             /**************************
 494              * CALCULATE INTERACTIONS *
 495              **************************/
 496
 497             r10              = _mm_mul_ps(rsq10,rinv10);
 498             r10              = _mm_andnot_ps(dummy_mask,r10);
 499
 500             /* Compute parameters for interactions between i and j atoms */
 501             qq10             = _mm_mul_ps(iq1,jq0);
 502
 503             /* EWALD ELECTROSTATICS */
 504
 505             /* Analytical PME correction */
 506             zeta2            = _mm_mul_ps(beta2,rsq10);
 507             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 508             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 509             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 510             felec            = _mm_mul_ps(qq10,felec);
 511             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 512             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv10);
 513             velec            = _mm_mul_ps(qq10,velec);
 514
 515             /* Update potential sum for this i atom from the interaction with this j atom. */
 516             velec            = _mm_andnot_ps(dummy_mask,velec);
 517             velecsum         = _mm_add_ps(velecsum,velec);
 518
 519             fscal            = felec;
 520
 521             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 522
 523              /* Update vectorial force */
 524             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 525             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 526             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 527
 528             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 529             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 530             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 531
 532             /**************************
 533              * CALCULATE INTERACTIONS *
 534              **************************/
 535
 536             r20              = _mm_mul_ps(rsq20,rinv20);
 537             r20              = _mm_andnot_ps(dummy_mask,r20);
 538
 539             /* Compute parameters for interactions between i and j atoms */
 540             qq20             = _mm_mul_ps(iq2,jq0);
 541
 542             /* EWALD ELECTROSTATICS */
 543
 544             /* Analytical PME correction */
 545             zeta2            = _mm_mul_ps(beta2,rsq20);
 546             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 547             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 548             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 549             felec            = _mm_mul_ps(qq20,felec);
 550             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 551             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv20);
 552             velec            = _mm_mul_ps(qq20,velec);
 553
 554             /* Update potential sum for this i atom from the interaction with this j atom. */
 555             velec            = _mm_andnot_ps(dummy_mask,velec);
 556             velecsum         = _mm_add_ps(velecsum,velec);
 557
 558             fscal            = felec;
 559
 560             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 561
 562              /* Update vectorial force */
 563             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 564             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 565             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 566
 567             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 568             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 569             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 570
 571             /**************************
 572              * CALCULATE INTERACTIONS *
 573              **************************/
 574
 575             r30              = _mm_mul_ps(rsq30,rinv30);
 576             r30              = _mm_andnot_ps(dummy_mask,r30);
 577
 578             /* Compute parameters for interactions between i and j atoms */
 579             qq30             = _mm_mul_ps(iq3,jq0);
 580
 581             /* EWALD ELECTROSTATICS */
 582
 583             /* Analytical PME correction */
 584             zeta2            = _mm_mul_ps(beta2,rsq30);
 585             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 586             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 587             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 588             felec            = _mm_mul_ps(qq30,felec);
 589             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 590             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv30);
 591             velec            = _mm_mul_ps(qq30,velec);
 592
 593             /* Update potential sum for this i atom from the interaction with this j atom. */
 594             velec            = _mm_andnot_ps(dummy_mask,velec);
 595             velecsum         = _mm_add_ps(velecsum,velec);
 596
 597             fscal            = felec;
 598
 599             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 600
 601              /* Update vectorial force */
 602             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 603             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 604             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 605
 606             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 607             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 608             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 609
 610             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
 611             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
 612             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
 613             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
 614
 615             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 616
 617             /* Inner loop uses 125 flops */
 618         }
 619
 620         /* End of innermost loop */
 621
 622         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
 623                                               f+i_coord_offset,fshift+i_shift_offset);
 624
 625         ggid                        = gid[iidx];
 626         /* Update potential energies */
 627         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
 628         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
 629
 630         /* Increment number of inner iterations */
 631         inneriter                  += j_index_end - j_index_start;
 632
 633         /* Outer loop uses 26 flops */
 634     }
 635
 636     /* Increment number of outer iterations */
 637     outeriter        += nri;
 638
 639     /* Update outer/inner flops */
 640
 641     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_VF,outeriter*26 + inneriter*125);
 642 }
 643 /*
 644  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4P1_F_avx_128_fma_single
 645  * Electrostatics interaction: Ewald
 646  * VdW interaction:            LennardJones
 647  * Geometry:                   Water4-Particle
 648  * Calculate force/pot:        Force
 649  */
 650 void
 651 nb_kernel_ElecEw_VdwLJ_GeomW4P1_F_avx_128_fma_single
 652                     (t_nblist * gmx_restrict                nlist,
 653                      rvec * gmx_restrict                    xx,
 654                      rvec * gmx_restrict                    ff,
 655                      t_forcerec * gmx_restrict              fr,
 656                      t_mdatoms * gmx_restrict               mdatoms,
 657                      nb_kernel_data_t * gmx_restrict        kernel_data,
 658                      t_nrnb * gmx_restrict                  nrnb)
 659 {
 660     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 661      * just 0 for non-waters.
 662      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
 663      * jnr indices corresponding to data put in the four positions in the SIMD register.
 664      */
 665     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 666     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 667     int              jnrA,jnrB,jnrC,jnrD;
 668     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
 669     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
 670     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 671     real             rcutoff_scalar;
 672     real             *shiftvec,*fshift,*x,*f;
 673     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
 674     real             scratch[4*DIM];
 675     __m128           fscal,rcutoff,rcutoff2,jidxall;
 676     int              vdwioffset0;
 677     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 678     int              vdwioffset1;
 679     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 680     int              vdwioffset2;
 681     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 682     int              vdwioffset3;
 683     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
 684     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
 685     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 686     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 687     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 688     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 689     __m128           dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
 690     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 691     real             *charge;
 692     int              nvdwtype;
 693     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 694     int              *vdwtype;
 695     real             *vdwparam;
 696     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 697     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 698     __m128i          ewitab;
 699     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
 700     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
 701     real             *ewtab;
 702     __m128           dummy_mask,cutoff_mask;
 703     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 704     __m128           one     = _mm_set1_ps(1.0);
 705     __m128           two     = _mm_set1_ps(2.0);
 706     x                = xx[0];
 707     f                = ff[0];
 708
 709     nri              = nlist->nri;
 710     iinr             = nlist->iinr;
 711     jindex           = nlist->jindex;
 712     jjnr             = nlist->jjnr;
 713     shiftidx         = nlist->shift;
 714     gid              = nlist->gid;
 715     shiftvec         = fr->shift_vec[0];
 716     fshift           = fr->fshift[0];
 717     facel            = _mm_set1_ps(fr->epsfac);
 718     charge           = mdatoms->chargeA;
 719     nvdwtype         = fr->ntype;
 720     vdwparam         = fr->nbfp;
 721     vdwtype          = mdatoms->typeA;
 722
 723     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
 724     beta             = _mm_set1_ps(fr->ic->ewaldcoeff);
 725     beta2            = _mm_mul_ps(beta,beta);
 726     beta3            = _mm_mul_ps(beta,beta2);
 727     ewtab            = fr->ic->tabq_coul_F;
 728     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
 729     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
 730
 731     /* Setup water-specific parameters */
 732     inr              = nlist->iinr[0];
 733     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 734     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 735     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
 736     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 737
 738     /* Avoid stupid compiler warnings */
 739     jnrA = jnrB = jnrC = jnrD = 0;
 740     j_coord_offsetA = 0;
 741     j_coord_offsetB = 0;
 742     j_coord_offsetC = 0;
 743     j_coord_offsetD = 0;
 744
 745     outeriter        = 0;
 746     inneriter        = 0;
 747
 748     for(iidx=0;iidx<4*DIM;iidx++)
 749     {
 750         scratch[iidx] = 0.0;
 751     }
 752
 753     /* Start outer loop over neighborlists */
 754     for(iidx=0; iidx<nri; iidx++)
 755     {
 756         /* Load shift vector for this list */
 757         i_shift_offset   = DIM*shiftidx[iidx];
 758
 759         /* Load limits for loop over neighbors */
 760         j_index_start    = jindex[iidx];
 761         j_index_end      = jindex[iidx+1];
 762
 763         /* Get outer coordinate index */
 764         inr              = iinr[iidx];
 765         i_coord_offset   = DIM*inr;
 766
 767         /* Load i particle coords and add shift vector */
 768         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 769                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 770
 771         fix0             = _mm_setzero_ps();
 772         fiy0             = _mm_setzero_ps();
 773         fiz0             = _mm_setzero_ps();
 774         fix1             = _mm_setzero_ps();
 775         fiy1             = _mm_setzero_ps();
 776         fiz1             = _mm_setzero_ps();
 777         fix2             = _mm_setzero_ps();
 778         fiy2             = _mm_setzero_ps();
 779         fiz2             = _mm_setzero_ps();
 780         fix3             = _mm_setzero_ps();
 781         fiy3             = _mm_setzero_ps();
 782         fiz3             = _mm_setzero_ps();
 783
 784         /* Start inner kernel loop */
 785         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 786         {
 787
 788             /* Get j neighbor index, and coordinate index */
 789             jnrA             = jjnr[jidx];
 790             jnrB             = jjnr[jidx+1];
 791             jnrC             = jjnr[jidx+2];
 792             jnrD             = jjnr[jidx+3];
 793             j_coord_offsetA  = DIM*jnrA;
 794             j_coord_offsetB  = DIM*jnrB;
 795             j_coord_offsetC  = DIM*jnrC;
 796             j_coord_offsetD  = DIM*jnrD;
 797
 798             /* load j atom coordinates */
 799             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 800                                               x+j_coord_offsetC,x+j_coord_offsetD,
 801                                               &jx0,&jy0,&jz0);
 802
 803             /* Calculate displacement vector */
 804             dx00             = _mm_sub_ps(ix0,jx0);
 805             dy00             = _mm_sub_ps(iy0,jy0);
 806             dz00             = _mm_sub_ps(iz0,jz0);
 807             dx10             = _mm_sub_ps(ix1,jx0);
 808             dy10             = _mm_sub_ps(iy1,jy0);
 809             dz10             = _mm_sub_ps(iz1,jz0);
 810             dx20             = _mm_sub_ps(ix2,jx0);
 811             dy20             = _mm_sub_ps(iy2,jy0);
 812             dz20             = _mm_sub_ps(iz2,jz0);
 813             dx30             = _mm_sub_ps(ix3,jx0);
 814             dy30             = _mm_sub_ps(iy3,jy0);
 815             dz30             = _mm_sub_ps(iz3,jz0);
 816
 817             /* Calculate squared distance and things based on it */
 818             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 819             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 820             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 821             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 822
 823             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 824             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 825             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 826
 827             rinvsq00         = gmx_mm_inv_ps(rsq00);
 828             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 829             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 830             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 831
 832             /* Load parameters for j particles */
 833             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 834                                                               charge+jnrC+0,charge+jnrD+0);
 835             vdwjidx0A        = 2*vdwtype[jnrA+0];
 836             vdwjidx0B        = 2*vdwtype[jnrB+0];
 837             vdwjidx0C        = 2*vdwtype[jnrC+0];
 838             vdwjidx0D        = 2*vdwtype[jnrD+0];
 839
 840             fjx0             = _mm_setzero_ps();
 841             fjy0             = _mm_setzero_ps();
 842             fjz0             = _mm_setzero_ps();
 843
 844             /**************************
 845              * CALCULATE INTERACTIONS *
 846              **************************/
 847
 848             /* Compute parameters for interactions between i and j atoms */
 849             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 850                                          vdwparam+vdwioffset0+vdwjidx0B,
 851                                          vdwparam+vdwioffset0+vdwjidx0C,
 852                                          vdwparam+vdwioffset0+vdwjidx0D,
 853                                          &c6_00,&c12_00);
 854
 855             /* LENNARD-JONES DISPERSION/REPULSION */
 856
 857             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 858             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
 859
 860             fscal            = fvdw;
 861
 862              /* Update vectorial force */
 863             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 864             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 865             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 866
 867             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 868             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 869             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 870
 871             /**************************
 872              * CALCULATE INTERACTIONS *
 873              **************************/
 874
 875             r10              = _mm_mul_ps(rsq10,rinv10);
 876
 877             /* Compute parameters for interactions between i and j atoms */
 878             qq10             = _mm_mul_ps(iq1,jq0);
 879
 880             /* EWALD ELECTROSTATICS */
 881
 882             /* Analytical PME correction */
 883             zeta2            = _mm_mul_ps(beta2,rsq10);
 884             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 885             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 886             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 887             felec            = _mm_mul_ps(qq10,felec);
 888
 889             fscal            = felec;
 890
 891              /* Update vectorial force */
 892             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 893             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 894             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 895
 896             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 897             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 898             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 899
 900             /**************************
 901              * CALCULATE INTERACTIONS *
 902              **************************/
 903
 904             r20              = _mm_mul_ps(rsq20,rinv20);
 905
 906             /* Compute parameters for interactions between i and j atoms */
 907             qq20             = _mm_mul_ps(iq2,jq0);
 908
 909             /* EWALD ELECTROSTATICS */
 910
 911             /* Analytical PME correction */
 912             zeta2            = _mm_mul_ps(beta2,rsq20);
 913             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 914             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 915             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 916             felec            = _mm_mul_ps(qq20,felec);
 917
 918             fscal            = felec;
 919
 920              /* Update vectorial force */
 921             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 922             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 923             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 924
 925             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 926             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 927             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 928
 929             /**************************
 930              * CALCULATE INTERACTIONS *
 931              **************************/
 932
 933             r30              = _mm_mul_ps(rsq30,rinv30);
 934
 935             /* Compute parameters for interactions between i and j atoms */
 936             qq30             = _mm_mul_ps(iq3,jq0);
 937
 938             /* EWALD ELECTROSTATICS */
 939
 940             /* Analytical PME correction */
 941             zeta2            = _mm_mul_ps(beta2,rsq30);
 942             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 943             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 944             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 945             felec            = _mm_mul_ps(qq30,felec);
 946
 947             fscal            = felec;
 948
 949              /* Update vectorial force */
 950             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 951             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 952             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 953
 954             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 955             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 956             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 957
 958             fjptrA             = f+j_coord_offsetA;
 959             fjptrB             = f+j_coord_offsetB;
 960             fjptrC             = f+j_coord_offsetC;
 961             fjptrD             = f+j_coord_offsetD;
 962
 963             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 964
 965             /* Inner loop uses 114 flops */
 966         }
 967
 968         if(jidx<j_index_end)
 969         {
 970
 971             /* Get j neighbor index, and coordinate index */
 972             jnrlistA         = jjnr[jidx];
 973             jnrlistB         = jjnr[jidx+1];
 974             jnrlistC         = jjnr[jidx+2];
 975             jnrlistD         = jjnr[jidx+3];
 976             /* Sign of each element will be negative for non-real atoms.
 977              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 978              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 979              */
 980             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 981             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 982             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 983             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 984             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 985             j_coord_offsetA  = DIM*jnrA;
 986             j_coord_offsetB  = DIM*jnrB;
 987             j_coord_offsetC  = DIM*jnrC;
 988             j_coord_offsetD  = DIM*jnrD;
 989
 990             /* load j atom coordinates */
 991             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 992                                               x+j_coord_offsetC,x+j_coord_offsetD,
 993                                               &jx0,&jy0,&jz0);
 994
 995             /* Calculate displacement vector */
 996             dx00             = _mm_sub_ps(ix0,jx0);
 997             dy00             = _mm_sub_ps(iy0,jy0);
 998             dz00             = _mm_sub_ps(iz0,jz0);
 999             dx10             = _mm_sub_ps(ix1,jx0);
1000             dy10             = _mm_sub_ps(iy1,jy0);
1001             dz10             = _mm_sub_ps(iz1,jz0);
1002             dx20             = _mm_sub_ps(ix2,jx0);
1003             dy20             = _mm_sub_ps(iy2,jy0);
1004             dz20             = _mm_sub_ps(iz2,jz0);
1005             dx30             = _mm_sub_ps(ix3,jx0);
1006             dy30             = _mm_sub_ps(iy3,jy0);
1007             dz30             = _mm_sub_ps(iz3,jz0);
1008
1009             /* Calculate squared distance and things based on it */
1010             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1011             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1012             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1013             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
1014
1015             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1016             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1017             rinv30           = gmx_mm_invsqrt_ps(rsq30);
1018
1019             rinvsq00         = gmx_mm_inv_ps(rsq00);
1020             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1021             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1022             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
1023
1024             /* Load parameters for j particles */
1025             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
1026                                                               charge+jnrC+0,charge+jnrD+0);
1027             vdwjidx0A        = 2*vdwtype[jnrA+0];
1028             vdwjidx0B        = 2*vdwtype[jnrB+0];
1029             vdwjidx0C        = 2*vdwtype[jnrC+0];
1030             vdwjidx0D        = 2*vdwtype[jnrD+0];
1031
1032             fjx0             = _mm_setzero_ps();
1033             fjy0             = _mm_setzero_ps();
1034             fjz0             = _mm_setzero_ps();
1035
1036             /**************************
1037              * CALCULATE INTERACTIONS *
1038              **************************/
1039
1040             /* Compute parameters for interactions between i and j atoms */
1041             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
1042                                          vdwparam+vdwioffset0+vdwjidx0B,
1043                                          vdwparam+vdwioffset0+vdwjidx0C,
1044                                          vdwparam+vdwioffset0+vdwjidx0D,
1045                                          &c6_00,&c12_00);
1046
1047             /* LENNARD-JONES DISPERSION/REPULSION */
1048
1049             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1050             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1051
1052             fscal            = fvdw;
1053
1054             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1055
1056              /* Update vectorial force */
1057             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1058             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1059             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1060
1061             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1062             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1063             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1064
1065             /**************************
1066              * CALCULATE INTERACTIONS *
1067              **************************/
1068
1069             r10              = _mm_mul_ps(rsq10,rinv10);
1070             r10              = _mm_andnot_ps(dummy_mask,r10);
1071
1072             /* Compute parameters for interactions between i and j atoms */
1073             qq10             = _mm_mul_ps(iq1,jq0);
1074
1075             /* EWALD ELECTROSTATICS */
1076
1077             /* Analytical PME correction */
1078             zeta2            = _mm_mul_ps(beta2,rsq10);
1079             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
1080             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1081             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1082             felec            = _mm_mul_ps(qq10,felec);
1083
1084             fscal            = felec;
1085
1086             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1087
1088              /* Update vectorial force */
1089             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1090             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1091             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1092
1093             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1094             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1095             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1096
1097             /**************************
1098              * CALCULATE INTERACTIONS *
1099              **************************/
1100
1101             r20              = _mm_mul_ps(rsq20,rinv20);
1102             r20              = _mm_andnot_ps(dummy_mask,r20);
1103
1104             /* Compute parameters for interactions between i and j atoms */
1105             qq20             = _mm_mul_ps(iq2,jq0);
1106
1107             /* EWALD ELECTROSTATICS */
1108
1109             /* Analytical PME correction */
1110             zeta2            = _mm_mul_ps(beta2,rsq20);
1111             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
1112             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1113             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1114             felec            = _mm_mul_ps(qq20,felec);
1115
1116             fscal            = felec;
1117
1118             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1119
1120              /* Update vectorial force */
1121             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1122             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1123             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1124
1125             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1126             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1127             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1128
1129             /**************************
1130              * CALCULATE INTERACTIONS *
1131              **************************/
1132
1133             r30              = _mm_mul_ps(rsq30,rinv30);
1134             r30              = _mm_andnot_ps(dummy_mask,r30);
1135
1136             /* Compute parameters for interactions between i and j atoms */
1137             qq30             = _mm_mul_ps(iq3,jq0);
1138
1139             /* EWALD ELECTROSTATICS */
1140
1141             /* Analytical PME correction */
1142             zeta2            = _mm_mul_ps(beta2,rsq30);
1143             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
1144             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1145             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1146             felec            = _mm_mul_ps(qq30,felec);
1147
1148             fscal            = felec;
1149
1150             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1151
1152              /* Update vectorial force */
1153             fix3             = _mm_macc_ps(dx30,fscal,fix3);
1154             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
1155             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
1156
1157             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
1158             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
1159             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
1160
1161             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1162             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1163             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1164             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1165
1166             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1167
1168             /* Inner loop uses 117 flops */
1169         }
1170
1171         /* End of innermost loop */
1172
1173         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1174                                               f+i_coord_offset,fshift+i_shift_offset);
1175
1176         /* Increment number of inner iterations */
1177         inneriter                  += j_index_end - j_index_start;
1178
1179         /* Outer loop uses 24 flops */
1180     }
1181
1182     /* Increment number of outer iterations */
1183     outeriter        += nri;
1184
1185     /* Update outer/inner flops */
1186
1187     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_F,outeriter*24 + inneriter*117);
1188 }