src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecEw_VdwLJ_GeomW4P1_avx_128_fma_single.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
  37  */
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42 #include <math.h>
  43
  44 #include "../nb_kernel.h"
  45 #include "types/simple.h"
  46 #include "vec.h"
  47 #include "nrnb.h"
  48
  49 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
  50 #include "kernelutil_x86_avx_128_fma_single.h"
  51
  52 /*
  53  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4P1_VF_avx_128_fma_single
  54  * Electrostatics interaction: Ewald
  55  * VdW interaction:            LennardJones
  56  * Geometry:                   Water4-Particle
  57  * Calculate force/pot:        PotentialAndForce
  58  */
  59 void
  60 nb_kernel_ElecEw_VdwLJ_GeomW4P1_VF_avx_128_fma_single
  61                     (t_nblist                    * gmx_restrict       nlist,
  62                      rvec                        * gmx_restrict          xx,
  63                      rvec                        * gmx_restrict          ff,
  64                      t_forcerec                  * gmx_restrict          fr,
  65                      t_mdatoms                   * gmx_restrict     mdatoms,
  66                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  67                      t_nrnb                      * gmx_restrict        nrnb)
  68 {
  69     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  70      * just 0 for non-waters.
  71      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  72      * jnr indices corresponding to data put in the four positions in the SIMD register.
  73      */
  74     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  75     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  76     int              jnrA,jnrB,jnrC,jnrD;
  77     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  78     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  79     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  80     real             rcutoff_scalar;
  81     real             *shiftvec,*fshift,*x,*f;
  82     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  83     real             scratch[4*DIM];
  84     __m128           fscal,rcutoff,rcutoff2,jidxall;
  85     int              vdwioffset0;
  86     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  87     int              vdwioffset1;
  88     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  89     int              vdwioffset2;
  90     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  91     int              vdwioffset3;
  92     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
  93     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  94     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  95     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  96     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
  97     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
  98     __m128           dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
  99     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 100     real             *charge;
 101     int              nvdwtype;
 102     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 103     int              *vdwtype;
 104     real             *vdwparam;
 105     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 106     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 107     __m128i          ewitab;
 108     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
 109     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
 110     real             *ewtab;
 111     __m128           dummy_mask,cutoff_mask;
 112     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 113     __m128           one     = _mm_set1_ps(1.0);
 114     __m128           two     = _mm_set1_ps(2.0);
 115     x                = xx[0];
 116     f                = ff[0];
 117
 118     nri              = nlist->nri;
 119     iinr             = nlist->iinr;
 120     jindex           = nlist->jindex;
 121     jjnr             = nlist->jjnr;
 122     shiftidx         = nlist->shift;
 123     gid              = nlist->gid;
 124     shiftvec         = fr->shift_vec[0];
 125     fshift           = fr->fshift[0];
 126     facel            = _mm_set1_ps(fr->epsfac);
 127     charge           = mdatoms->chargeA;
 128     nvdwtype         = fr->ntype;
 129     vdwparam         = fr->nbfp;
 130     vdwtype          = mdatoms->typeA;
 131
 132     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
 133     beta             = _mm_set1_ps(fr->ic->ewaldcoeff_q);
 134     beta2            = _mm_mul_ps(beta,beta);
 135     beta3            = _mm_mul_ps(beta,beta2);
 136     ewtab            = fr->ic->tabq_coul_FDV0;
 137     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
 138     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
 139
 140     /* Setup water-specific parameters */
 141     inr              = nlist->iinr[0];
 142     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 143     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 144     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
 145     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 146
 147     /* Avoid stupid compiler warnings */
 148     jnrA = jnrB = jnrC = jnrD = 0;
 149     j_coord_offsetA = 0;
 150     j_coord_offsetB = 0;
 151     j_coord_offsetC = 0;
 152     j_coord_offsetD = 0;
 153
 154     outeriter        = 0;
 155     inneriter        = 0;
 156
 157     for(iidx=0;iidx<4*DIM;iidx++)
 158     {
 159         scratch[iidx] = 0.0;
 160     }
 161
 162     /* Start outer loop over neighborlists */
 163     for(iidx=0; iidx<nri; iidx++)
 164     {
 165         /* Load shift vector for this list */
 166         i_shift_offset   = DIM*shiftidx[iidx];
 167
 168         /* Load limits for loop over neighbors */
 169         j_index_start    = jindex[iidx];
 170         j_index_end      = jindex[iidx+1];
 171
 172         /* Get outer coordinate index */
 173         inr              = iinr[iidx];
 174         i_coord_offset   = DIM*inr;
 175
 176         /* Load i particle coords and add shift vector */
 177         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 178                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 179
 180         fix0             = _mm_setzero_ps();
 181         fiy0             = _mm_setzero_ps();
 182         fiz0             = _mm_setzero_ps();
 183         fix1             = _mm_setzero_ps();
 184         fiy1             = _mm_setzero_ps();
 185         fiz1             = _mm_setzero_ps();
 186         fix2             = _mm_setzero_ps();
 187         fiy2             = _mm_setzero_ps();
 188         fiz2             = _mm_setzero_ps();
 189         fix3             = _mm_setzero_ps();
 190         fiy3             = _mm_setzero_ps();
 191         fiz3             = _mm_setzero_ps();
 192
 193         /* Reset potential sums */
 194         velecsum         = _mm_setzero_ps();
 195         vvdwsum          = _mm_setzero_ps();
 196
 197         /* Start inner kernel loop */
 198         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 199         {
 200
 201             /* Get j neighbor index, and coordinate index */
 202             jnrA             = jjnr[jidx];
 203             jnrB             = jjnr[jidx+1];
 204             jnrC             = jjnr[jidx+2];
 205             jnrD             = jjnr[jidx+3];
 206             j_coord_offsetA  = DIM*jnrA;
 207             j_coord_offsetB  = DIM*jnrB;
 208             j_coord_offsetC  = DIM*jnrC;
 209             j_coord_offsetD  = DIM*jnrD;
 210
 211             /* load j atom coordinates */
 212             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 213                                               x+j_coord_offsetC,x+j_coord_offsetD,
 214                                               &jx0,&jy0,&jz0);
 215
 216             /* Calculate displacement vector */
 217             dx00             = _mm_sub_ps(ix0,jx0);
 218             dy00             = _mm_sub_ps(iy0,jy0);
 219             dz00             = _mm_sub_ps(iz0,jz0);
 220             dx10             = _mm_sub_ps(ix1,jx0);
 221             dy10             = _mm_sub_ps(iy1,jy0);
 222             dz10             = _mm_sub_ps(iz1,jz0);
 223             dx20             = _mm_sub_ps(ix2,jx0);
 224             dy20             = _mm_sub_ps(iy2,jy0);
 225             dz20             = _mm_sub_ps(iz2,jz0);
 226             dx30             = _mm_sub_ps(ix3,jx0);
 227             dy30             = _mm_sub_ps(iy3,jy0);
 228             dz30             = _mm_sub_ps(iz3,jz0);
 229
 230             /* Calculate squared distance and things based on it */
 231             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 232             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 233             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 234             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 235
 236             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 237             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 238             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 239
 240             rinvsq00         = gmx_mm_inv_ps(rsq00);
 241             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 242             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 243             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 244
 245             /* Load parameters for j particles */
 246             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 247                                                               charge+jnrC+0,charge+jnrD+0);
 248             vdwjidx0A        = 2*vdwtype[jnrA+0];
 249             vdwjidx0B        = 2*vdwtype[jnrB+0];
 250             vdwjidx0C        = 2*vdwtype[jnrC+0];
 251             vdwjidx0D        = 2*vdwtype[jnrD+0];
 252
 253             fjx0             = _mm_setzero_ps();
 254             fjy0             = _mm_setzero_ps();
 255             fjz0             = _mm_setzero_ps();
 256
 257             /**************************
 258              * CALCULATE INTERACTIONS *
 259              **************************/
 260
 261             /* Compute parameters for interactions between i and j atoms */
 262             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 263                                          vdwparam+vdwioffset0+vdwjidx0B,
 264                                          vdwparam+vdwioffset0+vdwjidx0C,
 265                                          vdwparam+vdwioffset0+vdwjidx0D,
 266                                          &c6_00,&c12_00);
 267
 268             /* LENNARD-JONES DISPERSION/REPULSION */
 269
 270             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 271             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 272             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 273             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 274             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 275
 276             /* Update potential sum for this i atom from the interaction with this j atom. */
 277             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 278
 279             fscal            = fvdw;
 280
 281              /* Update vectorial force */
 282             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 283             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 284             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 285
 286             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 287             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 288             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 289
 290             /**************************
 291              * CALCULATE INTERACTIONS *
 292              **************************/
 293
 294             r10              = _mm_mul_ps(rsq10,rinv10);
 295
 296             /* Compute parameters for interactions between i and j atoms */
 297             qq10             = _mm_mul_ps(iq1,jq0);
 298
 299             /* EWALD ELECTROSTATICS */
 300
 301             /* Analytical PME correction */
 302             zeta2            = _mm_mul_ps(beta2,rsq10);
 303             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 304             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 305             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 306             felec            = _mm_mul_ps(qq10,felec);
 307             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 308             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv10);
 309             velec            = _mm_mul_ps(qq10,velec);
 310
 311             /* Update potential sum for this i atom from the interaction with this j atom. */
 312             velecsum         = _mm_add_ps(velecsum,velec);
 313
 314             fscal            = felec;
 315
 316              /* Update vectorial force */
 317             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 318             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 319             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 320
 321             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 322             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 323             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 324
 325             /**************************
 326              * CALCULATE INTERACTIONS *
 327              **************************/
 328
 329             r20              = _mm_mul_ps(rsq20,rinv20);
 330
 331             /* Compute parameters for interactions between i and j atoms */
 332             qq20             = _mm_mul_ps(iq2,jq0);
 333
 334             /* EWALD ELECTROSTATICS */
 335
 336             /* Analytical PME correction */
 337             zeta2            = _mm_mul_ps(beta2,rsq20);
 338             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 339             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 340             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 341             felec            = _mm_mul_ps(qq20,felec);
 342             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 343             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv20);
 344             velec            = _mm_mul_ps(qq20,velec);
 345
 346             /* Update potential sum for this i atom from the interaction with this j atom. */
 347             velecsum         = _mm_add_ps(velecsum,velec);
 348
 349             fscal            = felec;
 350
 351              /* Update vectorial force */
 352             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 353             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 354             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 355
 356             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 357             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 358             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 359
 360             /**************************
 361              * CALCULATE INTERACTIONS *
 362              **************************/
 363
 364             r30              = _mm_mul_ps(rsq30,rinv30);
 365
 366             /* Compute parameters for interactions between i and j atoms */
 367             qq30             = _mm_mul_ps(iq3,jq0);
 368
 369             /* EWALD ELECTROSTATICS */
 370
 371             /* Analytical PME correction */
 372             zeta2            = _mm_mul_ps(beta2,rsq30);
 373             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 374             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 375             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 376             felec            = _mm_mul_ps(qq30,felec);
 377             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 378             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv30);
 379             velec            = _mm_mul_ps(qq30,velec);
 380
 381             /* Update potential sum for this i atom from the interaction with this j atom. */
 382             velecsum         = _mm_add_ps(velecsum,velec);
 383
 384             fscal            = felec;
 385
 386              /* Update vectorial force */
 387             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 388             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 389             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 390
 391             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 392             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 393             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 394
 395             fjptrA             = f+j_coord_offsetA;
 396             fjptrB             = f+j_coord_offsetB;
 397             fjptrC             = f+j_coord_offsetC;
 398             fjptrD             = f+j_coord_offsetD;
 399
 400             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 401
 402             /* Inner loop uses 122 flops */
 403         }
 404
 405         if(jidx<j_index_end)
 406         {
 407
 408             /* Get j neighbor index, and coordinate index */
 409             jnrlistA         = jjnr[jidx];
 410             jnrlistB         = jjnr[jidx+1];
 411             jnrlistC         = jjnr[jidx+2];
 412             jnrlistD         = jjnr[jidx+3];
 413             /* Sign of each element will be negative for non-real atoms.
 414              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 415              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 416              */
 417             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 418             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 419             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 420             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 421             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 422             j_coord_offsetA  = DIM*jnrA;
 423             j_coord_offsetB  = DIM*jnrB;
 424             j_coord_offsetC  = DIM*jnrC;
 425             j_coord_offsetD  = DIM*jnrD;
 426
 427             /* load j atom coordinates */
 428             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 429                                               x+j_coord_offsetC,x+j_coord_offsetD,
 430                                               &jx0,&jy0,&jz0);
 431
 432             /* Calculate displacement vector */
 433             dx00             = _mm_sub_ps(ix0,jx0);
 434             dy00             = _mm_sub_ps(iy0,jy0);
 435             dz00             = _mm_sub_ps(iz0,jz0);
 436             dx10             = _mm_sub_ps(ix1,jx0);
 437             dy10             = _mm_sub_ps(iy1,jy0);
 438             dz10             = _mm_sub_ps(iz1,jz0);
 439             dx20             = _mm_sub_ps(ix2,jx0);
 440             dy20             = _mm_sub_ps(iy2,jy0);
 441             dz20             = _mm_sub_ps(iz2,jz0);
 442             dx30             = _mm_sub_ps(ix3,jx0);
 443             dy30             = _mm_sub_ps(iy3,jy0);
 444             dz30             = _mm_sub_ps(iz3,jz0);
 445
 446             /* Calculate squared distance and things based on it */
 447             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 448             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 449             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 450             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 451
 452             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 453             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 454             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 455
 456             rinvsq00         = gmx_mm_inv_ps(rsq00);
 457             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 458             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 459             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 460
 461             /* Load parameters for j particles */
 462             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 463                                                               charge+jnrC+0,charge+jnrD+0);
 464             vdwjidx0A        = 2*vdwtype[jnrA+0];
 465             vdwjidx0B        = 2*vdwtype[jnrB+0];
 466             vdwjidx0C        = 2*vdwtype[jnrC+0];
 467             vdwjidx0D        = 2*vdwtype[jnrD+0];
 468
 469             fjx0             = _mm_setzero_ps();
 470             fjy0             = _mm_setzero_ps();
 471             fjz0             = _mm_setzero_ps();
 472
 473             /**************************
 474              * CALCULATE INTERACTIONS *
 475              **************************/
 476
 477             /* Compute parameters for interactions between i and j atoms */
 478             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 479                                          vdwparam+vdwioffset0+vdwjidx0B,
 480                                          vdwparam+vdwioffset0+vdwjidx0C,
 481                                          vdwparam+vdwioffset0+vdwjidx0D,
 482                                          &c6_00,&c12_00);
 483
 484             /* LENNARD-JONES DISPERSION/REPULSION */
 485
 486             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 487             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 488             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 489             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 490             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 491
 492             /* Update potential sum for this i atom from the interaction with this j atom. */
 493             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 494             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 495
 496             fscal            = fvdw;
 497
 498             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 499
 500              /* Update vectorial force */
 501             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 502             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 503             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 504
 505             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 506             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 507             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 508
 509             /**************************
 510              * CALCULATE INTERACTIONS *
 511              **************************/
 512
 513             r10              = _mm_mul_ps(rsq10,rinv10);
 514             r10              = _mm_andnot_ps(dummy_mask,r10);
 515
 516             /* Compute parameters for interactions between i and j atoms */
 517             qq10             = _mm_mul_ps(iq1,jq0);
 518
 519             /* EWALD ELECTROSTATICS */
 520
 521             /* Analytical PME correction */
 522             zeta2            = _mm_mul_ps(beta2,rsq10);
 523             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 524             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 525             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 526             felec            = _mm_mul_ps(qq10,felec);
 527             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 528             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv10);
 529             velec            = _mm_mul_ps(qq10,velec);
 530
 531             /* Update potential sum for this i atom from the interaction with this j atom. */
 532             velec            = _mm_andnot_ps(dummy_mask,velec);
 533             velecsum         = _mm_add_ps(velecsum,velec);
 534
 535             fscal            = felec;
 536
 537             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 538
 539              /* Update vectorial force */
 540             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 541             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 542             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 543
 544             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 545             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 546             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 547
 548             /**************************
 549              * CALCULATE INTERACTIONS *
 550              **************************/
 551
 552             r20              = _mm_mul_ps(rsq20,rinv20);
 553             r20              = _mm_andnot_ps(dummy_mask,r20);
 554
 555             /* Compute parameters for interactions between i and j atoms */
 556             qq20             = _mm_mul_ps(iq2,jq0);
 557
 558             /* EWALD ELECTROSTATICS */
 559
 560             /* Analytical PME correction */
 561             zeta2            = _mm_mul_ps(beta2,rsq20);
 562             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 563             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 564             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 565             felec            = _mm_mul_ps(qq20,felec);
 566             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 567             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv20);
 568             velec            = _mm_mul_ps(qq20,velec);
 569
 570             /* Update potential sum for this i atom from the interaction with this j atom. */
 571             velec            = _mm_andnot_ps(dummy_mask,velec);
 572             velecsum         = _mm_add_ps(velecsum,velec);
 573
 574             fscal            = felec;
 575
 576             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 577
 578              /* Update vectorial force */
 579             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 580             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 581             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 582
 583             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 584             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 585             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 586
 587             /**************************
 588              * CALCULATE INTERACTIONS *
 589              **************************/
 590
 591             r30              = _mm_mul_ps(rsq30,rinv30);
 592             r30              = _mm_andnot_ps(dummy_mask,r30);
 593
 594             /* Compute parameters for interactions between i and j atoms */
 595             qq30             = _mm_mul_ps(iq3,jq0);
 596
 597             /* EWALD ELECTROSTATICS */
 598
 599             /* Analytical PME correction */
 600             zeta2            = _mm_mul_ps(beta2,rsq30);
 601             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 602             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 603             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 604             felec            = _mm_mul_ps(qq30,felec);
 605             pmecorrV         = gmx_mm_pmecorrV_ps(zeta2);
 606             velec            = _mm_nmacc_ps(pmecorrV,beta,rinv30);
 607             velec            = _mm_mul_ps(qq30,velec);
 608
 609             /* Update potential sum for this i atom from the interaction with this j atom. */
 610             velec            = _mm_andnot_ps(dummy_mask,velec);
 611             velecsum         = _mm_add_ps(velecsum,velec);
 612
 613             fscal            = felec;
 614
 615             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 616
 617              /* Update vectorial force */
 618             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 619             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 620             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 621
 622             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 623             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 624             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 625
 626             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
 627             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
 628             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
 629             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
 630
 631             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 632
 633             /* Inner loop uses 125 flops */
 634         }
 635
 636         /* End of innermost loop */
 637
 638         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
 639                                               f+i_coord_offset,fshift+i_shift_offset);
 640
 641         ggid                        = gid[iidx];
 642         /* Update potential energies */
 643         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
 644         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
 645
 646         /* Increment number of inner iterations */
 647         inneriter                  += j_index_end - j_index_start;
 648
 649         /* Outer loop uses 26 flops */
 650     }
 651
 652     /* Increment number of outer iterations */
 653     outeriter        += nri;
 654
 655     /* Update outer/inner flops */
 656
 657     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_VF,outeriter*26 + inneriter*125);
 658 }
 659 /*
 660  * Gromacs nonbonded kernel:   nb_kernel_ElecEw_VdwLJ_GeomW4P1_F_avx_128_fma_single
 661  * Electrostatics interaction: Ewald
 662  * VdW interaction:            LennardJones
 663  * Geometry:                   Water4-Particle
 664  * Calculate force/pot:        Force
 665  */
 666 void
 667 nb_kernel_ElecEw_VdwLJ_GeomW4P1_F_avx_128_fma_single
 668                     (t_nblist                    * gmx_restrict       nlist,
 669                      rvec                        * gmx_restrict          xx,
 670                      rvec                        * gmx_restrict          ff,
 671                      t_forcerec                  * gmx_restrict          fr,
 672                      t_mdatoms                   * gmx_restrict     mdatoms,
 673                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
 674                      t_nrnb                      * gmx_restrict        nrnb)
 675 {
 676     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 677      * just 0 for non-waters.
 678      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
 679      * jnr indices corresponding to data put in the four positions in the SIMD register.
 680      */
 681     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 682     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 683     int              jnrA,jnrB,jnrC,jnrD;
 684     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
 685     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
 686     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 687     real             rcutoff_scalar;
 688     real             *shiftvec,*fshift,*x,*f;
 689     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
 690     real             scratch[4*DIM];
 691     __m128           fscal,rcutoff,rcutoff2,jidxall;
 692     int              vdwioffset0;
 693     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 694     int              vdwioffset1;
 695     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 696     int              vdwioffset2;
 697     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 698     int              vdwioffset3;
 699     __m128           ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
 700     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
 701     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 702     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 703     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 704     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 705     __m128           dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
 706     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 707     real             *charge;
 708     int              nvdwtype;
 709     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 710     int              *vdwtype;
 711     real             *vdwparam;
 712     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 713     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 714     __m128i          ewitab;
 715     __m128           ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
 716     __m128           beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
 717     real             *ewtab;
 718     __m128           dummy_mask,cutoff_mask;
 719     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 720     __m128           one     = _mm_set1_ps(1.0);
 721     __m128           two     = _mm_set1_ps(2.0);
 722     x                = xx[0];
 723     f                = ff[0];
 724
 725     nri              = nlist->nri;
 726     iinr             = nlist->iinr;
 727     jindex           = nlist->jindex;
 728     jjnr             = nlist->jjnr;
 729     shiftidx         = nlist->shift;
 730     gid              = nlist->gid;
 731     shiftvec         = fr->shift_vec[0];
 732     fshift           = fr->fshift[0];
 733     facel            = _mm_set1_ps(fr->epsfac);
 734     charge           = mdatoms->chargeA;
 735     nvdwtype         = fr->ntype;
 736     vdwparam         = fr->nbfp;
 737     vdwtype          = mdatoms->typeA;
 738
 739     sh_ewald         = _mm_set1_ps(fr->ic->sh_ewald);
 740     beta             = _mm_set1_ps(fr->ic->ewaldcoeff_q);
 741     beta2            = _mm_mul_ps(beta,beta);
 742     beta3            = _mm_mul_ps(beta,beta2);
 743     ewtab            = fr->ic->tabq_coul_F;
 744     ewtabscale       = _mm_set1_ps(fr->ic->tabq_scale);
 745     ewtabhalfspace   = _mm_set1_ps(0.5/fr->ic->tabq_scale);
 746
 747     /* Setup water-specific parameters */
 748     inr              = nlist->iinr[0];
 749     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 750     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 751     iq3              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
 752     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 753
 754     /* Avoid stupid compiler warnings */
 755     jnrA = jnrB = jnrC = jnrD = 0;
 756     j_coord_offsetA = 0;
 757     j_coord_offsetB = 0;
 758     j_coord_offsetC = 0;
 759     j_coord_offsetD = 0;
 760
 761     outeriter        = 0;
 762     inneriter        = 0;
 763
 764     for(iidx=0;iidx<4*DIM;iidx++)
 765     {
 766         scratch[iidx] = 0.0;
 767     }
 768
 769     /* Start outer loop over neighborlists */
 770     for(iidx=0; iidx<nri; iidx++)
 771     {
 772         /* Load shift vector for this list */
 773         i_shift_offset   = DIM*shiftidx[iidx];
 774
 775         /* Load limits for loop over neighbors */
 776         j_index_start    = jindex[iidx];
 777         j_index_end      = jindex[iidx+1];
 778
 779         /* Get outer coordinate index */
 780         inr              = iinr[iidx];
 781         i_coord_offset   = DIM*inr;
 782
 783         /* Load i particle coords and add shift vector */
 784         gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 785                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
 786
 787         fix0             = _mm_setzero_ps();
 788         fiy0             = _mm_setzero_ps();
 789         fiz0             = _mm_setzero_ps();
 790         fix1             = _mm_setzero_ps();
 791         fiy1             = _mm_setzero_ps();
 792         fiz1             = _mm_setzero_ps();
 793         fix2             = _mm_setzero_ps();
 794         fiy2             = _mm_setzero_ps();
 795         fiz2             = _mm_setzero_ps();
 796         fix3             = _mm_setzero_ps();
 797         fiy3             = _mm_setzero_ps();
 798         fiz3             = _mm_setzero_ps();
 799
 800         /* Start inner kernel loop */
 801         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 802         {
 803
 804             /* Get j neighbor index, and coordinate index */
 805             jnrA             = jjnr[jidx];
 806             jnrB             = jjnr[jidx+1];
 807             jnrC             = jjnr[jidx+2];
 808             jnrD             = jjnr[jidx+3];
 809             j_coord_offsetA  = DIM*jnrA;
 810             j_coord_offsetB  = DIM*jnrB;
 811             j_coord_offsetC  = DIM*jnrC;
 812             j_coord_offsetD  = DIM*jnrD;
 813
 814             /* load j atom coordinates */
 815             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 816                                               x+j_coord_offsetC,x+j_coord_offsetD,
 817                                               &jx0,&jy0,&jz0);
 818
 819             /* Calculate displacement vector */
 820             dx00             = _mm_sub_ps(ix0,jx0);
 821             dy00             = _mm_sub_ps(iy0,jy0);
 822             dz00             = _mm_sub_ps(iz0,jz0);
 823             dx10             = _mm_sub_ps(ix1,jx0);
 824             dy10             = _mm_sub_ps(iy1,jy0);
 825             dz10             = _mm_sub_ps(iz1,jz0);
 826             dx20             = _mm_sub_ps(ix2,jx0);
 827             dy20             = _mm_sub_ps(iy2,jy0);
 828             dz20             = _mm_sub_ps(iz2,jz0);
 829             dx30             = _mm_sub_ps(ix3,jx0);
 830             dy30             = _mm_sub_ps(iy3,jy0);
 831             dz30             = _mm_sub_ps(iz3,jz0);
 832
 833             /* Calculate squared distance and things based on it */
 834             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 835             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 836             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 837             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
 838
 839             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 840             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 841             rinv30           = gmx_mm_invsqrt_ps(rsq30);
 842
 843             rinvsq00         = gmx_mm_inv_ps(rsq00);
 844             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 845             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 846             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
 847
 848             /* Load parameters for j particles */
 849             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
 850                                                               charge+jnrC+0,charge+jnrD+0);
 851             vdwjidx0A        = 2*vdwtype[jnrA+0];
 852             vdwjidx0B        = 2*vdwtype[jnrB+0];
 853             vdwjidx0C        = 2*vdwtype[jnrC+0];
 854             vdwjidx0D        = 2*vdwtype[jnrD+0];
 855
 856             fjx0             = _mm_setzero_ps();
 857             fjy0             = _mm_setzero_ps();
 858             fjz0             = _mm_setzero_ps();
 859
 860             /**************************
 861              * CALCULATE INTERACTIONS *
 862              **************************/
 863
 864             /* Compute parameters for interactions between i and j atoms */
 865             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
 866                                          vdwparam+vdwioffset0+vdwjidx0B,
 867                                          vdwparam+vdwioffset0+vdwjidx0C,
 868                                          vdwparam+vdwioffset0+vdwjidx0D,
 869                                          &c6_00,&c12_00);
 870
 871             /* LENNARD-JONES DISPERSION/REPULSION */
 872
 873             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 874             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
 875
 876             fscal            = fvdw;
 877
 878              /* Update vectorial force */
 879             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 880             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 881             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 882
 883             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 884             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 885             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 886
 887             /**************************
 888              * CALCULATE INTERACTIONS *
 889              **************************/
 890
 891             r10              = _mm_mul_ps(rsq10,rinv10);
 892
 893             /* Compute parameters for interactions between i and j atoms */
 894             qq10             = _mm_mul_ps(iq1,jq0);
 895
 896             /* EWALD ELECTROSTATICS */
 897
 898             /* Analytical PME correction */
 899             zeta2            = _mm_mul_ps(beta2,rsq10);
 900             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
 901             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 902             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 903             felec            = _mm_mul_ps(qq10,felec);
 904
 905             fscal            = felec;
 906
 907              /* Update vectorial force */
 908             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 909             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 910             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 911
 912             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 913             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 914             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 915
 916             /**************************
 917              * CALCULATE INTERACTIONS *
 918              **************************/
 919
 920             r20              = _mm_mul_ps(rsq20,rinv20);
 921
 922             /* Compute parameters for interactions between i and j atoms */
 923             qq20             = _mm_mul_ps(iq2,jq0);
 924
 925             /* EWALD ELECTROSTATICS */
 926
 927             /* Analytical PME correction */
 928             zeta2            = _mm_mul_ps(beta2,rsq20);
 929             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
 930             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 931             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 932             felec            = _mm_mul_ps(qq20,felec);
 933
 934             fscal            = felec;
 935
 936              /* Update vectorial force */
 937             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 938             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 939             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 940
 941             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 942             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 943             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 944
 945             /**************************
 946              * CALCULATE INTERACTIONS *
 947              **************************/
 948
 949             r30              = _mm_mul_ps(rsq30,rinv30);
 950
 951             /* Compute parameters for interactions between i and j atoms */
 952             qq30             = _mm_mul_ps(iq3,jq0);
 953
 954             /* EWALD ELECTROSTATICS */
 955
 956             /* Analytical PME correction */
 957             zeta2            = _mm_mul_ps(beta2,rsq30);
 958             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
 959             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
 960             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
 961             felec            = _mm_mul_ps(qq30,felec);
 962
 963             fscal            = felec;
 964
 965              /* Update vectorial force */
 966             fix3             = _mm_macc_ps(dx30,fscal,fix3);
 967             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
 968             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
 969
 970             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
 971             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
 972             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
 973
 974             fjptrA             = f+j_coord_offsetA;
 975             fjptrB             = f+j_coord_offsetB;
 976             fjptrC             = f+j_coord_offsetC;
 977             fjptrD             = f+j_coord_offsetD;
 978
 979             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
 980
 981             /* Inner loop uses 114 flops */
 982         }
 983
 984         if(jidx<j_index_end)
 985         {
 986
 987             /* Get j neighbor index, and coordinate index */
 988             jnrlistA         = jjnr[jidx];
 989             jnrlistB         = jjnr[jidx+1];
 990             jnrlistC         = jjnr[jidx+2];
 991             jnrlistD         = jjnr[jidx+3];
 992             /* Sign of each element will be negative for non-real atoms.
 993              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 994              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 995              */
 996             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 997             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 998             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 999             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1000             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1001             j_coord_offsetA  = DIM*jnrA;
1002             j_coord_offsetB  = DIM*jnrB;
1003             j_coord_offsetC  = DIM*jnrC;
1004             j_coord_offsetD  = DIM*jnrD;
1005
1006             /* load j atom coordinates */
1007             gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1008                                               x+j_coord_offsetC,x+j_coord_offsetD,
1009                                               &jx0,&jy0,&jz0);
1010
1011             /* Calculate displacement vector */
1012             dx00             = _mm_sub_ps(ix0,jx0);
1013             dy00             = _mm_sub_ps(iy0,jy0);
1014             dz00             = _mm_sub_ps(iz0,jz0);
1015             dx10             = _mm_sub_ps(ix1,jx0);
1016             dy10             = _mm_sub_ps(iy1,jy0);
1017             dz10             = _mm_sub_ps(iz1,jz0);
1018             dx20             = _mm_sub_ps(ix2,jx0);
1019             dy20             = _mm_sub_ps(iy2,jy0);
1020             dz20             = _mm_sub_ps(iz2,jz0);
1021             dx30             = _mm_sub_ps(ix3,jx0);
1022             dy30             = _mm_sub_ps(iy3,jy0);
1023             dz30             = _mm_sub_ps(iz3,jz0);
1024
1025             /* Calculate squared distance and things based on it */
1026             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1027             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1028             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1029             rsq30            = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
1030
1031             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1032             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1033             rinv30           = gmx_mm_invsqrt_ps(rsq30);
1034
1035             rinvsq00         = gmx_mm_inv_ps(rsq00);
1036             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1037             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1038             rinvsq30         = _mm_mul_ps(rinv30,rinv30);
1039
1040             /* Load parameters for j particles */
1041             jq0              = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
1042                                                               charge+jnrC+0,charge+jnrD+0);
1043             vdwjidx0A        = 2*vdwtype[jnrA+0];
1044             vdwjidx0B        = 2*vdwtype[jnrB+0];
1045             vdwjidx0C        = 2*vdwtype[jnrC+0];
1046             vdwjidx0D        = 2*vdwtype[jnrD+0];
1047
1048             fjx0             = _mm_setzero_ps();
1049             fjy0             = _mm_setzero_ps();
1050             fjz0             = _mm_setzero_ps();
1051
1052             /**************************
1053              * CALCULATE INTERACTIONS *
1054              **************************/
1055
1056             /* Compute parameters for interactions between i and j atoms */
1057             gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
1058                                          vdwparam+vdwioffset0+vdwjidx0B,
1059                                          vdwparam+vdwioffset0+vdwjidx0C,
1060                                          vdwparam+vdwioffset0+vdwjidx0D,
1061                                          &c6_00,&c12_00);
1062
1063             /* LENNARD-JONES DISPERSION/REPULSION */
1064
1065             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1066             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1067
1068             fscal            = fvdw;
1069
1070             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1071
1072              /* Update vectorial force */
1073             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1074             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1075             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1076
1077             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1078             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1079             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1080
1081             /**************************
1082              * CALCULATE INTERACTIONS *
1083              **************************/
1084
1085             r10              = _mm_mul_ps(rsq10,rinv10);
1086             r10              = _mm_andnot_ps(dummy_mask,r10);
1087
1088             /* Compute parameters for interactions between i and j atoms */
1089             qq10             = _mm_mul_ps(iq1,jq0);
1090
1091             /* EWALD ELECTROSTATICS */
1092
1093             /* Analytical PME correction */
1094             zeta2            = _mm_mul_ps(beta2,rsq10);
1095             rinv3            = _mm_mul_ps(rinvsq10,rinv10);
1096             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1097             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1098             felec            = _mm_mul_ps(qq10,felec);
1099
1100             fscal            = felec;
1101
1102             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1103
1104              /* Update vectorial force */
1105             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1106             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1107             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1108
1109             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1110             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1111             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1112
1113             /**************************
1114              * CALCULATE INTERACTIONS *
1115              **************************/
1116
1117             r20              = _mm_mul_ps(rsq20,rinv20);
1118             r20              = _mm_andnot_ps(dummy_mask,r20);
1119
1120             /* Compute parameters for interactions between i and j atoms */
1121             qq20             = _mm_mul_ps(iq2,jq0);
1122
1123             /* EWALD ELECTROSTATICS */
1124
1125             /* Analytical PME correction */
1126             zeta2            = _mm_mul_ps(beta2,rsq20);
1127             rinv3            = _mm_mul_ps(rinvsq20,rinv20);
1128             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1129             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1130             felec            = _mm_mul_ps(qq20,felec);
1131
1132             fscal            = felec;
1133
1134             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1135
1136              /* Update vectorial force */
1137             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1138             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1139             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1140
1141             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1142             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1143             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1144
1145             /**************************
1146              * CALCULATE INTERACTIONS *
1147              **************************/
1148
1149             r30              = _mm_mul_ps(rsq30,rinv30);
1150             r30              = _mm_andnot_ps(dummy_mask,r30);
1151
1152             /* Compute parameters for interactions between i and j atoms */
1153             qq30             = _mm_mul_ps(iq3,jq0);
1154
1155             /* EWALD ELECTROSTATICS */
1156
1157             /* Analytical PME correction */
1158             zeta2            = _mm_mul_ps(beta2,rsq30);
1159             rinv3            = _mm_mul_ps(rinvsq30,rinv30);
1160             pmecorrF         = gmx_mm_pmecorrF_ps(zeta2);
1161             felec            = _mm_macc_ps(pmecorrF,beta3,rinv3);
1162             felec            = _mm_mul_ps(qq30,felec);
1163
1164             fscal            = felec;
1165
1166             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1167
1168              /* Update vectorial force */
1169             fix3             = _mm_macc_ps(dx30,fscal,fix3);
1170             fiy3             = _mm_macc_ps(dy30,fscal,fiy3);
1171             fiz3             = _mm_macc_ps(dz30,fscal,fiz3);
1172
1173             fjx0             = _mm_macc_ps(dx30,fscal,fjx0);
1174             fjy0             = _mm_macc_ps(dy30,fscal,fjy0);
1175             fjz0             = _mm_macc_ps(dz30,fscal,fjz0);
1176
1177             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1178             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1179             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1180             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1181
1182             gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1183
1184             /* Inner loop uses 117 flops */
1185         }
1186
1187         /* End of innermost loop */
1188
1189         gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1190                                               f+i_coord_offset,fshift+i_shift_offset);
1191
1192         /* Increment number of inner iterations */
1193         inneriter                  += j_index_end - j_index_start;
1194
1195         /* Outer loop uses 24 flops */
1196     }
1197
1198     /* Increment number of outer iterations */
1199     outeriter        += nri;
1200
1201     /* Update outer/inner flops */
1202
1203     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_F,outeriter*24 + inneriter*117);
1204 }