src/gromacs/gmxlib/nonbonded/nb_kernel_avx_128_fma_single/nb_kernel_ElecRF_VdwLJ_GeomW3W3_avx_128_fma_single.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2012,2013, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*
  36  * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
  37  */
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42 #include <math.h>
  43
  44 #include "../nb_kernel.h"
  45 #include "types/simple.h"
  46 #include "vec.h"
  47 #include "nrnb.h"
  48
  49 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
  50 #include "kernelutil_x86_avx_128_fma_single.h"
  51
  52 /*
  53  * Gromacs nonbonded kernel:   nb_kernel_ElecRF_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  54  * Electrostatics interaction: ReactionField
  55  * VdW interaction:            LennardJones
  56  * Geometry:                   Water3-Water3
  57  * Calculate force/pot:        PotentialAndForce
  58  */
  59 void
  60 nb_kernel_ElecRF_VdwLJ_GeomW3W3_VF_avx_128_fma_single
  61                     (t_nblist                    * gmx_restrict       nlist,
  62                      rvec                        * gmx_restrict          xx,
  63                      rvec                        * gmx_restrict          ff,
  64                      t_forcerec                  * gmx_restrict          fr,
  65                      t_mdatoms                   * gmx_restrict     mdatoms,
  66                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
  67                      t_nrnb                      * gmx_restrict        nrnb)
  68 {
  69     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
  70      * just 0 for non-waters.
  71      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
  72      * jnr indices corresponding to data put in the four positions in the SIMD register.
  73      */
  74     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
  75     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
  76     int              jnrA,jnrB,jnrC,jnrD;
  77     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
  78     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
  79     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
  80     real             rcutoff_scalar;
  81     real             *shiftvec,*fshift,*x,*f;
  82     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
  83     real             scratch[4*DIM];
  84     __m128           fscal,rcutoff,rcutoff2,jidxall;
  85     int              vdwioffset0;
  86     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
  87     int              vdwioffset1;
  88     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
  89     int              vdwioffset2;
  90     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
  91     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
  92     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
  93     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
  94     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
  95     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
  96     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
  97     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
  98     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
  99     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
 100     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 101     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 102     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 103     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 104     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 105     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 106     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 107     real             *charge;
 108     int              nvdwtype;
 109     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 110     int              *vdwtype;
 111     real             *vdwparam;
 112     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 113     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 114     __m128           dummy_mask,cutoff_mask;
 115     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 116     __m128           one     = _mm_set1_ps(1.0);
 117     __m128           two     = _mm_set1_ps(2.0);
 118     x                = xx[0];
 119     f                = ff[0];
 120
 121     nri              = nlist->nri;
 122     iinr             = nlist->iinr;
 123     jindex           = nlist->jindex;
 124     jjnr             = nlist->jjnr;
 125     shiftidx         = nlist->shift;
 126     gid              = nlist->gid;
 127     shiftvec         = fr->shift_vec[0];
 128     fshift           = fr->fshift[0];
 129     facel            = _mm_set1_ps(fr->epsfac);
 130     charge           = mdatoms->chargeA;
 131     krf              = _mm_set1_ps(fr->ic->k_rf);
 132     krf2             = _mm_set1_ps(fr->ic->k_rf*2.0);
 133     crf              = _mm_set1_ps(fr->ic->c_rf);
 134     nvdwtype         = fr->ntype;
 135     vdwparam         = fr->nbfp;
 136     vdwtype          = mdatoms->typeA;
 137
 138     /* Setup water-specific parameters */
 139     inr              = nlist->iinr[0];
 140     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 141     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 142     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 143     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 144
 145     jq0              = _mm_set1_ps(charge[inr+0]);
 146     jq1              = _mm_set1_ps(charge[inr+1]);
 147     jq2              = _mm_set1_ps(charge[inr+2]);
 148     vdwjidx0A        = 2*vdwtype[inr+0];
 149     qq00             = _mm_mul_ps(iq0,jq0);
 150     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 151     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 152     qq01             = _mm_mul_ps(iq0,jq1);
 153     qq02             = _mm_mul_ps(iq0,jq2);
 154     qq10             = _mm_mul_ps(iq1,jq0);
 155     qq11             = _mm_mul_ps(iq1,jq1);
 156     qq12             = _mm_mul_ps(iq1,jq2);
 157     qq20             = _mm_mul_ps(iq2,jq0);
 158     qq21             = _mm_mul_ps(iq2,jq1);
 159     qq22             = _mm_mul_ps(iq2,jq2);
 160
 161     /* Avoid stupid compiler warnings */
 162     jnrA = jnrB = jnrC = jnrD = 0;
 163     j_coord_offsetA = 0;
 164     j_coord_offsetB = 0;
 165     j_coord_offsetC = 0;
 166     j_coord_offsetD = 0;
 167
 168     outeriter        = 0;
 169     inneriter        = 0;
 170
 171     for(iidx=0;iidx<4*DIM;iidx++)
 172     {
 173         scratch[iidx] = 0.0;
 174     }
 175
 176     /* Start outer loop over neighborlists */
 177     for(iidx=0; iidx<nri; iidx++)
 178     {
 179         /* Load shift vector for this list */
 180         i_shift_offset   = DIM*shiftidx[iidx];
 181
 182         /* Load limits for loop over neighbors */
 183         j_index_start    = jindex[iidx];
 184         j_index_end      = jindex[iidx+1];
 185
 186         /* Get outer coordinate index */
 187         inr              = iinr[iidx];
 188         i_coord_offset   = DIM*inr;
 189
 190         /* Load i particle coords and add shift vector */
 191         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
 192                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
 193
 194         fix0             = _mm_setzero_ps();
 195         fiy0             = _mm_setzero_ps();
 196         fiz0             = _mm_setzero_ps();
 197         fix1             = _mm_setzero_ps();
 198         fiy1             = _mm_setzero_ps();
 199         fiz1             = _mm_setzero_ps();
 200         fix2             = _mm_setzero_ps();
 201         fiy2             = _mm_setzero_ps();
 202         fiz2             = _mm_setzero_ps();
 203
 204         /* Reset potential sums */
 205         velecsum         = _mm_setzero_ps();
 206         vvdwsum          = _mm_setzero_ps();
 207
 208         /* Start inner kernel loop */
 209         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
 210         {
 211
 212             /* Get j neighbor index, and coordinate index */
 213             jnrA             = jjnr[jidx];
 214             jnrB             = jjnr[jidx+1];
 215             jnrC             = jjnr[jidx+2];
 216             jnrD             = jjnr[jidx+3];
 217             j_coord_offsetA  = DIM*jnrA;
 218             j_coord_offsetB  = DIM*jnrB;
 219             j_coord_offsetC  = DIM*jnrC;
 220             j_coord_offsetD  = DIM*jnrD;
 221
 222             /* load j atom coordinates */
 223             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 224                                               x+j_coord_offsetC,x+j_coord_offsetD,
 225                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 226
 227             /* Calculate displacement vector */
 228             dx00             = _mm_sub_ps(ix0,jx0);
 229             dy00             = _mm_sub_ps(iy0,jy0);
 230             dz00             = _mm_sub_ps(iz0,jz0);
 231             dx01             = _mm_sub_ps(ix0,jx1);
 232             dy01             = _mm_sub_ps(iy0,jy1);
 233             dz01             = _mm_sub_ps(iz0,jz1);
 234             dx02             = _mm_sub_ps(ix0,jx2);
 235             dy02             = _mm_sub_ps(iy0,jy2);
 236             dz02             = _mm_sub_ps(iz0,jz2);
 237             dx10             = _mm_sub_ps(ix1,jx0);
 238             dy10             = _mm_sub_ps(iy1,jy0);
 239             dz10             = _mm_sub_ps(iz1,jz0);
 240             dx11             = _mm_sub_ps(ix1,jx1);
 241             dy11             = _mm_sub_ps(iy1,jy1);
 242             dz11             = _mm_sub_ps(iz1,jz1);
 243             dx12             = _mm_sub_ps(ix1,jx2);
 244             dy12             = _mm_sub_ps(iy1,jy2);
 245             dz12             = _mm_sub_ps(iz1,jz2);
 246             dx20             = _mm_sub_ps(ix2,jx0);
 247             dy20             = _mm_sub_ps(iy2,jy0);
 248             dz20             = _mm_sub_ps(iz2,jz0);
 249             dx21             = _mm_sub_ps(ix2,jx1);
 250             dy21             = _mm_sub_ps(iy2,jy1);
 251             dz21             = _mm_sub_ps(iz2,jz1);
 252             dx22             = _mm_sub_ps(ix2,jx2);
 253             dy22             = _mm_sub_ps(iy2,jy2);
 254             dz22             = _mm_sub_ps(iz2,jz2);
 255
 256             /* Calculate squared distance and things based on it */
 257             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 258             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 259             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 260             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 261             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 262             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 263             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 264             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 265             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 266
 267             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 268             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 269             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 270             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 271             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 272             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 273             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 274             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 275             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 276
 277             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 278             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
 279             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
 280             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 281             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 282             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 283             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 284             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 285             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 286
 287             fjx0             = _mm_setzero_ps();
 288             fjy0             = _mm_setzero_ps();
 289             fjz0             = _mm_setzero_ps();
 290             fjx1             = _mm_setzero_ps();
 291             fjy1             = _mm_setzero_ps();
 292             fjz1             = _mm_setzero_ps();
 293             fjx2             = _mm_setzero_ps();
 294             fjy2             = _mm_setzero_ps();
 295             fjz2             = _mm_setzero_ps();
 296
 297             /**************************
 298              * CALCULATE INTERACTIONS *
 299              **************************/
 300
 301             /* REACTION-FIELD ELECTROSTATICS */
 302             velec            = _mm_mul_ps(qq00,_mm_sub_ps(_mm_macc_ps(krf,rsq00,rinv00),crf));
 303             felec            = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
 304
 305             /* LENNARD-JONES DISPERSION/REPULSION */
 306
 307             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 308             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 309             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 310             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 311             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 312
 313             /* Update potential sum for this i atom from the interaction with this j atom. */
 314             velecsum         = _mm_add_ps(velecsum,velec);
 315             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 316
 317             fscal            = _mm_add_ps(felec,fvdw);
 318
 319              /* Update vectorial force */
 320             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 321             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 322             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 323
 324             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 325             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 326             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 327
 328             /**************************
 329              * CALCULATE INTERACTIONS *
 330              **************************/
 331
 332             /* REACTION-FIELD ELECTROSTATICS */
 333             velec            = _mm_mul_ps(qq01,_mm_sub_ps(_mm_macc_ps(krf,rsq01,rinv01),crf));
 334             felec            = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
 335
 336             /* Update potential sum for this i atom from the interaction with this j atom. */
 337             velecsum         = _mm_add_ps(velecsum,velec);
 338
 339             fscal            = felec;
 340
 341              /* Update vectorial force */
 342             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 343             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 344             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 345
 346             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 347             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 348             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 349
 350             /**************************
 351              * CALCULATE INTERACTIONS *
 352              **************************/
 353
 354             /* REACTION-FIELD ELECTROSTATICS */
 355             velec            = _mm_mul_ps(qq02,_mm_sub_ps(_mm_macc_ps(krf,rsq02,rinv02),crf));
 356             felec            = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
 357
 358             /* Update potential sum for this i atom from the interaction with this j atom. */
 359             velecsum         = _mm_add_ps(velecsum,velec);
 360
 361             fscal            = felec;
 362
 363              /* Update vectorial force */
 364             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 365             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 366             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 367
 368             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 369             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 370             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 371
 372             /**************************
 373              * CALCULATE INTERACTIONS *
 374              **************************/
 375
 376             /* REACTION-FIELD ELECTROSTATICS */
 377             velec            = _mm_mul_ps(qq10,_mm_sub_ps(_mm_macc_ps(krf,rsq10,rinv10),crf));
 378             felec            = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
 379
 380             /* Update potential sum for this i atom from the interaction with this j atom. */
 381             velecsum         = _mm_add_ps(velecsum,velec);
 382
 383             fscal            = felec;
 384
 385              /* Update vectorial force */
 386             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 387             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 388             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 389
 390             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 391             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 392             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 393
 394             /**************************
 395              * CALCULATE INTERACTIONS *
 396              **************************/
 397
 398             /* REACTION-FIELD ELECTROSTATICS */
 399             velec            = _mm_mul_ps(qq11,_mm_sub_ps(_mm_macc_ps(krf,rsq11,rinv11),crf));
 400             felec            = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
 401
 402             /* Update potential sum for this i atom from the interaction with this j atom. */
 403             velecsum         = _mm_add_ps(velecsum,velec);
 404
 405             fscal            = felec;
 406
 407              /* Update vectorial force */
 408             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 409             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 410             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 411
 412             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 413             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 414             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 415
 416             /**************************
 417              * CALCULATE INTERACTIONS *
 418              **************************/
 419
 420             /* REACTION-FIELD ELECTROSTATICS */
 421             velec            = _mm_mul_ps(qq12,_mm_sub_ps(_mm_macc_ps(krf,rsq12,rinv12),crf));
 422             felec            = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
 423
 424             /* Update potential sum for this i atom from the interaction with this j atom. */
 425             velecsum         = _mm_add_ps(velecsum,velec);
 426
 427             fscal            = felec;
 428
 429              /* Update vectorial force */
 430             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 431             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 432             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 433
 434             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 435             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 436             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 437
 438             /**************************
 439              * CALCULATE INTERACTIONS *
 440              **************************/
 441
 442             /* REACTION-FIELD ELECTROSTATICS */
 443             velec            = _mm_mul_ps(qq20,_mm_sub_ps(_mm_macc_ps(krf,rsq20,rinv20),crf));
 444             felec            = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
 445
 446             /* Update potential sum for this i atom from the interaction with this j atom. */
 447             velecsum         = _mm_add_ps(velecsum,velec);
 448
 449             fscal            = felec;
 450
 451              /* Update vectorial force */
 452             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 453             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 454             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 455
 456             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 457             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 458             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 459
 460             /**************************
 461              * CALCULATE INTERACTIONS *
 462              **************************/
 463
 464             /* REACTION-FIELD ELECTROSTATICS */
 465             velec            = _mm_mul_ps(qq21,_mm_sub_ps(_mm_macc_ps(krf,rsq21,rinv21),crf));
 466             felec            = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
 467
 468             /* Update potential sum for this i atom from the interaction with this j atom. */
 469             velecsum         = _mm_add_ps(velecsum,velec);
 470
 471             fscal            = felec;
 472
 473              /* Update vectorial force */
 474             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 475             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 476             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 477
 478             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 479             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 480             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 481
 482             /**************************
 483              * CALCULATE INTERACTIONS *
 484              **************************/
 485
 486             /* REACTION-FIELD ELECTROSTATICS */
 487             velec            = _mm_mul_ps(qq22,_mm_sub_ps(_mm_macc_ps(krf,rsq22,rinv22),crf));
 488             felec            = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
 489
 490             /* Update potential sum for this i atom from the interaction with this j atom. */
 491             velecsum         = _mm_add_ps(velecsum,velec);
 492
 493             fscal            = felec;
 494
 495              /* Update vectorial force */
 496             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 497             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 498             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 499
 500             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 501             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 502             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 503
 504             fjptrA             = f+j_coord_offsetA;
 505             fjptrB             = f+j_coord_offsetB;
 506             fjptrC             = f+j_coord_offsetC;
 507             fjptrD             = f+j_coord_offsetD;
 508
 509             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 510                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 511
 512             /* Inner loop uses 327 flops */
 513         }
 514
 515         if(jidx<j_index_end)
 516         {
 517
 518             /* Get j neighbor index, and coordinate index */
 519             jnrlistA         = jjnr[jidx];
 520             jnrlistB         = jjnr[jidx+1];
 521             jnrlistC         = jjnr[jidx+2];
 522             jnrlistD         = jjnr[jidx+3];
 523             /* Sign of each element will be negative for non-real atoms.
 524              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
 525              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
 526              */
 527             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
 528             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
 529             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
 530             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
 531             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
 532             j_coord_offsetA  = DIM*jnrA;
 533             j_coord_offsetB  = DIM*jnrB;
 534             j_coord_offsetC  = DIM*jnrC;
 535             j_coord_offsetD  = DIM*jnrD;
 536
 537             /* load j atom coordinates */
 538             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
 539                                               x+j_coord_offsetC,x+j_coord_offsetD,
 540                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
 541
 542             /* Calculate displacement vector */
 543             dx00             = _mm_sub_ps(ix0,jx0);
 544             dy00             = _mm_sub_ps(iy0,jy0);
 545             dz00             = _mm_sub_ps(iz0,jz0);
 546             dx01             = _mm_sub_ps(ix0,jx1);
 547             dy01             = _mm_sub_ps(iy0,jy1);
 548             dz01             = _mm_sub_ps(iz0,jz1);
 549             dx02             = _mm_sub_ps(ix0,jx2);
 550             dy02             = _mm_sub_ps(iy0,jy2);
 551             dz02             = _mm_sub_ps(iz0,jz2);
 552             dx10             = _mm_sub_ps(ix1,jx0);
 553             dy10             = _mm_sub_ps(iy1,jy0);
 554             dz10             = _mm_sub_ps(iz1,jz0);
 555             dx11             = _mm_sub_ps(ix1,jx1);
 556             dy11             = _mm_sub_ps(iy1,jy1);
 557             dz11             = _mm_sub_ps(iz1,jz1);
 558             dx12             = _mm_sub_ps(ix1,jx2);
 559             dy12             = _mm_sub_ps(iy1,jy2);
 560             dz12             = _mm_sub_ps(iz1,jz2);
 561             dx20             = _mm_sub_ps(ix2,jx0);
 562             dy20             = _mm_sub_ps(iy2,jy0);
 563             dz20             = _mm_sub_ps(iz2,jz0);
 564             dx21             = _mm_sub_ps(ix2,jx1);
 565             dy21             = _mm_sub_ps(iy2,jy1);
 566             dz21             = _mm_sub_ps(iz2,jz1);
 567             dx22             = _mm_sub_ps(ix2,jx2);
 568             dy22             = _mm_sub_ps(iy2,jy2);
 569             dz22             = _mm_sub_ps(iz2,jz2);
 570
 571             /* Calculate squared distance and things based on it */
 572             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
 573             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
 574             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
 575             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
 576             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
 577             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
 578             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
 579             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
 580             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
 581
 582             rinv00           = gmx_mm_invsqrt_ps(rsq00);
 583             rinv01           = gmx_mm_invsqrt_ps(rsq01);
 584             rinv02           = gmx_mm_invsqrt_ps(rsq02);
 585             rinv10           = gmx_mm_invsqrt_ps(rsq10);
 586             rinv11           = gmx_mm_invsqrt_ps(rsq11);
 587             rinv12           = gmx_mm_invsqrt_ps(rsq12);
 588             rinv20           = gmx_mm_invsqrt_ps(rsq20);
 589             rinv21           = gmx_mm_invsqrt_ps(rsq21);
 590             rinv22           = gmx_mm_invsqrt_ps(rsq22);
 591
 592             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
 593             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
 594             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
 595             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
 596             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
 597             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
 598             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
 599             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
 600             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
 601
 602             fjx0             = _mm_setzero_ps();
 603             fjy0             = _mm_setzero_ps();
 604             fjz0             = _mm_setzero_ps();
 605             fjx1             = _mm_setzero_ps();
 606             fjy1             = _mm_setzero_ps();
 607             fjz1             = _mm_setzero_ps();
 608             fjx2             = _mm_setzero_ps();
 609             fjy2             = _mm_setzero_ps();
 610             fjz2             = _mm_setzero_ps();
 611
 612             /**************************
 613              * CALCULATE INTERACTIONS *
 614              **************************/
 615
 616             /* REACTION-FIELD ELECTROSTATICS */
 617             velec            = _mm_mul_ps(qq00,_mm_sub_ps(_mm_macc_ps(krf,rsq00,rinv00),crf));
 618             felec            = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
 619
 620             /* LENNARD-JONES DISPERSION/REPULSION */
 621
 622             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
 623             vvdw6            = _mm_mul_ps(c6_00,rinvsix);
 624             vvdw12           = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
 625             vvdw             = _mm_msub_ps(vvdw12,one_twelfth,_mm_mul_ps(vvdw6,one_sixth));
 626             fvdw             = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
 627
 628             /* Update potential sum for this i atom from the interaction with this j atom. */
 629             velec            = _mm_andnot_ps(dummy_mask,velec);
 630             velecsum         = _mm_add_ps(velecsum,velec);
 631             vvdw             = _mm_andnot_ps(dummy_mask,vvdw);
 632             vvdwsum          = _mm_add_ps(vvdwsum,vvdw);
 633
 634             fscal            = _mm_add_ps(felec,fvdw);
 635
 636             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 637
 638              /* Update vectorial force */
 639             fix0             = _mm_macc_ps(dx00,fscal,fix0);
 640             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
 641             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
 642
 643             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
 644             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
 645             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
 646
 647             /**************************
 648              * CALCULATE INTERACTIONS *
 649              **************************/
 650
 651             /* REACTION-FIELD ELECTROSTATICS */
 652             velec            = _mm_mul_ps(qq01,_mm_sub_ps(_mm_macc_ps(krf,rsq01,rinv01),crf));
 653             felec            = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
 654
 655             /* Update potential sum for this i atom from the interaction with this j atom. */
 656             velec            = _mm_andnot_ps(dummy_mask,velec);
 657             velecsum         = _mm_add_ps(velecsum,velec);
 658
 659             fscal            = felec;
 660
 661             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 662
 663              /* Update vectorial force */
 664             fix0             = _mm_macc_ps(dx01,fscal,fix0);
 665             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
 666             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
 667
 668             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
 669             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
 670             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
 671
 672             /**************************
 673              * CALCULATE INTERACTIONS *
 674              **************************/
 675
 676             /* REACTION-FIELD ELECTROSTATICS */
 677             velec            = _mm_mul_ps(qq02,_mm_sub_ps(_mm_macc_ps(krf,rsq02,rinv02),crf));
 678             felec            = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
 679
 680             /* Update potential sum for this i atom from the interaction with this j atom. */
 681             velec            = _mm_andnot_ps(dummy_mask,velec);
 682             velecsum         = _mm_add_ps(velecsum,velec);
 683
 684             fscal            = felec;
 685
 686             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 687
 688              /* Update vectorial force */
 689             fix0             = _mm_macc_ps(dx02,fscal,fix0);
 690             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
 691             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
 692
 693             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
 694             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
 695             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
 696
 697             /**************************
 698              * CALCULATE INTERACTIONS *
 699              **************************/
 700
 701             /* REACTION-FIELD ELECTROSTATICS */
 702             velec            = _mm_mul_ps(qq10,_mm_sub_ps(_mm_macc_ps(krf,rsq10,rinv10),crf));
 703             felec            = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
 704
 705             /* Update potential sum for this i atom from the interaction with this j atom. */
 706             velec            = _mm_andnot_ps(dummy_mask,velec);
 707             velecsum         = _mm_add_ps(velecsum,velec);
 708
 709             fscal            = felec;
 710
 711             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 712
 713              /* Update vectorial force */
 714             fix1             = _mm_macc_ps(dx10,fscal,fix1);
 715             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
 716             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
 717
 718             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
 719             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
 720             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
 721
 722             /**************************
 723              * CALCULATE INTERACTIONS *
 724              **************************/
 725
 726             /* REACTION-FIELD ELECTROSTATICS */
 727             velec            = _mm_mul_ps(qq11,_mm_sub_ps(_mm_macc_ps(krf,rsq11,rinv11),crf));
 728             felec            = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
 729
 730             /* Update potential sum for this i atom from the interaction with this j atom. */
 731             velec            = _mm_andnot_ps(dummy_mask,velec);
 732             velecsum         = _mm_add_ps(velecsum,velec);
 733
 734             fscal            = felec;
 735
 736             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 737
 738              /* Update vectorial force */
 739             fix1             = _mm_macc_ps(dx11,fscal,fix1);
 740             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
 741             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
 742
 743             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
 744             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
 745             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
 746
 747             /**************************
 748              * CALCULATE INTERACTIONS *
 749              **************************/
 750
 751             /* REACTION-FIELD ELECTROSTATICS */
 752             velec            = _mm_mul_ps(qq12,_mm_sub_ps(_mm_macc_ps(krf,rsq12,rinv12),crf));
 753             felec            = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
 754
 755             /* Update potential sum for this i atom from the interaction with this j atom. */
 756             velec            = _mm_andnot_ps(dummy_mask,velec);
 757             velecsum         = _mm_add_ps(velecsum,velec);
 758
 759             fscal            = felec;
 760
 761             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 762
 763              /* Update vectorial force */
 764             fix1             = _mm_macc_ps(dx12,fscal,fix1);
 765             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
 766             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
 767
 768             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
 769             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
 770             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
 771
 772             /**************************
 773              * CALCULATE INTERACTIONS *
 774              **************************/
 775
 776             /* REACTION-FIELD ELECTROSTATICS */
 777             velec            = _mm_mul_ps(qq20,_mm_sub_ps(_mm_macc_ps(krf,rsq20,rinv20),crf));
 778             felec            = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
 779
 780             /* Update potential sum for this i atom from the interaction with this j atom. */
 781             velec            = _mm_andnot_ps(dummy_mask,velec);
 782             velecsum         = _mm_add_ps(velecsum,velec);
 783
 784             fscal            = felec;
 785
 786             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 787
 788              /* Update vectorial force */
 789             fix2             = _mm_macc_ps(dx20,fscal,fix2);
 790             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
 791             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
 792
 793             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
 794             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
 795             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
 796
 797             /**************************
 798              * CALCULATE INTERACTIONS *
 799              **************************/
 800
 801             /* REACTION-FIELD ELECTROSTATICS */
 802             velec            = _mm_mul_ps(qq21,_mm_sub_ps(_mm_macc_ps(krf,rsq21,rinv21),crf));
 803             felec            = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
 804
 805             /* Update potential sum for this i atom from the interaction with this j atom. */
 806             velec            = _mm_andnot_ps(dummy_mask,velec);
 807             velecsum         = _mm_add_ps(velecsum,velec);
 808
 809             fscal            = felec;
 810
 811             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 812
 813              /* Update vectorial force */
 814             fix2             = _mm_macc_ps(dx21,fscal,fix2);
 815             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
 816             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
 817
 818             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
 819             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
 820             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
 821
 822             /**************************
 823              * CALCULATE INTERACTIONS *
 824              **************************/
 825
 826             /* REACTION-FIELD ELECTROSTATICS */
 827             velec            = _mm_mul_ps(qq22,_mm_sub_ps(_mm_macc_ps(krf,rsq22,rinv22),crf));
 828             felec            = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
 829
 830             /* Update potential sum for this i atom from the interaction with this j atom. */
 831             velec            = _mm_andnot_ps(dummy_mask,velec);
 832             velecsum         = _mm_add_ps(velecsum,velec);
 833
 834             fscal            = felec;
 835
 836             fscal            = _mm_andnot_ps(dummy_mask,fscal);
 837
 838              /* Update vectorial force */
 839             fix2             = _mm_macc_ps(dx22,fscal,fix2);
 840             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
 841             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
 842
 843             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
 844             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
 845             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
 846
 847             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
 848             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
 849             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
 850             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
 851
 852             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
 853                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
 854
 855             /* Inner loop uses 327 flops */
 856         }
 857
 858         /* End of innermost loop */
 859
 860         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
 861                                               f+i_coord_offset,fshift+i_shift_offset);
 862
 863         ggid                        = gid[iidx];
 864         /* Update potential energies */
 865         gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
 866         gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
 867
 868         /* Increment number of inner iterations */
 869         inneriter                  += j_index_end - j_index_start;
 870
 871         /* Outer loop uses 20 flops */
 872     }
 873
 874     /* Increment number of outer iterations */
 875     outeriter        += nri;
 876
 877     /* Update outer/inner flops */
 878
 879     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*327);
 880 }
 881 /*
 882  * Gromacs nonbonded kernel:   nb_kernel_ElecRF_VdwLJ_GeomW3W3_F_avx_128_fma_single
 883  * Electrostatics interaction: ReactionField
 884  * VdW interaction:            LennardJones
 885  * Geometry:                   Water3-Water3
 886  * Calculate force/pot:        Force
 887  */
 888 void
 889 nb_kernel_ElecRF_VdwLJ_GeomW3W3_F_avx_128_fma_single
 890                     (t_nblist                    * gmx_restrict       nlist,
 891                      rvec                        * gmx_restrict          xx,
 892                      rvec                        * gmx_restrict          ff,
 893                      t_forcerec                  * gmx_restrict          fr,
 894                      t_mdatoms                   * gmx_restrict     mdatoms,
 895                      nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
 896                      t_nrnb                      * gmx_restrict        nrnb)
 897 {
 898     /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
 899      * just 0 for non-waters.
 900      * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
 901      * jnr indices corresponding to data put in the four positions in the SIMD register.
 902      */
 903     int              i_shift_offset,i_coord_offset,outeriter,inneriter;
 904     int              j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
 905     int              jnrA,jnrB,jnrC,jnrD;
 906     int              jnrlistA,jnrlistB,jnrlistC,jnrlistD;
 907     int              j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
 908     int              *iinr,*jindex,*jjnr,*shiftidx,*gid;
 909     real             rcutoff_scalar;
 910     real             *shiftvec,*fshift,*x,*f;
 911     real             *fjptrA,*fjptrB,*fjptrC,*fjptrD;
 912     real             scratch[4*DIM];
 913     __m128           fscal,rcutoff,rcutoff2,jidxall;
 914     int              vdwioffset0;
 915     __m128           ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
 916     int              vdwioffset1;
 917     __m128           ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
 918     int              vdwioffset2;
 919     __m128           ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
 920     int              vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
 921     __m128           jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
 922     int              vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
 923     __m128           jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
 924     int              vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
 925     __m128           jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
 926     __m128           dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
 927     __m128           dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
 928     __m128           dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
 929     __m128           dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
 930     __m128           dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
 931     __m128           dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
 932     __m128           dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
 933     __m128           dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
 934     __m128           dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
 935     __m128           velec,felec,velecsum,facel,crf,krf,krf2;
 936     real             *charge;
 937     int              nvdwtype;
 938     __m128           rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
 939     int              *vdwtype;
 940     real             *vdwparam;
 941     __m128           one_sixth   = _mm_set1_ps(1.0/6.0);
 942     __m128           one_twelfth = _mm_set1_ps(1.0/12.0);
 943     __m128           dummy_mask,cutoff_mask;
 944     __m128           signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
 945     __m128           one     = _mm_set1_ps(1.0);
 946     __m128           two     = _mm_set1_ps(2.0);
 947     x                = xx[0];
 948     f                = ff[0];
 949
 950     nri              = nlist->nri;
 951     iinr             = nlist->iinr;
 952     jindex           = nlist->jindex;
 953     jjnr             = nlist->jjnr;
 954     shiftidx         = nlist->shift;
 955     gid              = nlist->gid;
 956     shiftvec         = fr->shift_vec[0];
 957     fshift           = fr->fshift[0];
 958     facel            = _mm_set1_ps(fr->epsfac);
 959     charge           = mdatoms->chargeA;
 960     krf              = _mm_set1_ps(fr->ic->k_rf);
 961     krf2             = _mm_set1_ps(fr->ic->k_rf*2.0);
 962     crf              = _mm_set1_ps(fr->ic->c_rf);
 963     nvdwtype         = fr->ntype;
 964     vdwparam         = fr->nbfp;
 965     vdwtype          = mdatoms->typeA;
 966
 967     /* Setup water-specific parameters */
 968     inr              = nlist->iinr[0];
 969     iq0              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
 970     iq1              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
 971     iq2              = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
 972     vdwioffset0      = 2*nvdwtype*vdwtype[inr+0];
 973
 974     jq0              = _mm_set1_ps(charge[inr+0]);
 975     jq1              = _mm_set1_ps(charge[inr+1]);
 976     jq2              = _mm_set1_ps(charge[inr+2]);
 977     vdwjidx0A        = 2*vdwtype[inr+0];
 978     qq00             = _mm_mul_ps(iq0,jq0);
 979     c6_00            = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
 980     c12_00           = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
 981     qq01             = _mm_mul_ps(iq0,jq1);
 982     qq02             = _mm_mul_ps(iq0,jq2);
 983     qq10             = _mm_mul_ps(iq1,jq0);
 984     qq11             = _mm_mul_ps(iq1,jq1);
 985     qq12             = _mm_mul_ps(iq1,jq2);
 986     qq20             = _mm_mul_ps(iq2,jq0);
 987     qq21             = _mm_mul_ps(iq2,jq1);
 988     qq22             = _mm_mul_ps(iq2,jq2);
 989
 990     /* Avoid stupid compiler warnings */
 991     jnrA = jnrB = jnrC = jnrD = 0;
 992     j_coord_offsetA = 0;
 993     j_coord_offsetB = 0;
 994     j_coord_offsetC = 0;
 995     j_coord_offsetD = 0;
 996
 997     outeriter        = 0;
 998     inneriter        = 0;
 999
1000     for(iidx=0;iidx<4*DIM;iidx++)
1001     {
1002         scratch[iidx] = 0.0;
1003     }
1004
1005     /* Start outer loop over neighborlists */
1006     for(iidx=0; iidx<nri; iidx++)
1007     {
1008         /* Load shift vector for this list */
1009         i_shift_offset   = DIM*shiftidx[iidx];
1010
1011         /* Load limits for loop over neighbors */
1012         j_index_start    = jindex[iidx];
1013         j_index_end      = jindex[iidx+1];
1014
1015         /* Get outer coordinate index */
1016         inr              = iinr[iidx];
1017         i_coord_offset   = DIM*inr;
1018
1019         /* Load i particle coords and add shift vector */
1020         gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1021                                                  &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1022
1023         fix0             = _mm_setzero_ps();
1024         fiy0             = _mm_setzero_ps();
1025         fiz0             = _mm_setzero_ps();
1026         fix1             = _mm_setzero_ps();
1027         fiy1             = _mm_setzero_ps();
1028         fiz1             = _mm_setzero_ps();
1029         fix2             = _mm_setzero_ps();
1030         fiy2             = _mm_setzero_ps();
1031         fiz2             = _mm_setzero_ps();
1032
1033         /* Start inner kernel loop */
1034         for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1035         {
1036
1037             /* Get j neighbor index, and coordinate index */
1038             jnrA             = jjnr[jidx];
1039             jnrB             = jjnr[jidx+1];
1040             jnrC             = jjnr[jidx+2];
1041             jnrD             = jjnr[jidx+3];
1042             j_coord_offsetA  = DIM*jnrA;
1043             j_coord_offsetB  = DIM*jnrB;
1044             j_coord_offsetC  = DIM*jnrC;
1045             j_coord_offsetD  = DIM*jnrD;
1046
1047             /* load j atom coordinates */
1048             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1049                                               x+j_coord_offsetC,x+j_coord_offsetD,
1050                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1051
1052             /* Calculate displacement vector */
1053             dx00             = _mm_sub_ps(ix0,jx0);
1054             dy00             = _mm_sub_ps(iy0,jy0);
1055             dz00             = _mm_sub_ps(iz0,jz0);
1056             dx01             = _mm_sub_ps(ix0,jx1);
1057             dy01             = _mm_sub_ps(iy0,jy1);
1058             dz01             = _mm_sub_ps(iz0,jz1);
1059             dx02             = _mm_sub_ps(ix0,jx2);
1060             dy02             = _mm_sub_ps(iy0,jy2);
1061             dz02             = _mm_sub_ps(iz0,jz2);
1062             dx10             = _mm_sub_ps(ix1,jx0);
1063             dy10             = _mm_sub_ps(iy1,jy0);
1064             dz10             = _mm_sub_ps(iz1,jz0);
1065             dx11             = _mm_sub_ps(ix1,jx1);
1066             dy11             = _mm_sub_ps(iy1,jy1);
1067             dz11             = _mm_sub_ps(iz1,jz1);
1068             dx12             = _mm_sub_ps(ix1,jx2);
1069             dy12             = _mm_sub_ps(iy1,jy2);
1070             dz12             = _mm_sub_ps(iz1,jz2);
1071             dx20             = _mm_sub_ps(ix2,jx0);
1072             dy20             = _mm_sub_ps(iy2,jy0);
1073             dz20             = _mm_sub_ps(iz2,jz0);
1074             dx21             = _mm_sub_ps(ix2,jx1);
1075             dy21             = _mm_sub_ps(iy2,jy1);
1076             dz21             = _mm_sub_ps(iz2,jz1);
1077             dx22             = _mm_sub_ps(ix2,jx2);
1078             dy22             = _mm_sub_ps(iy2,jy2);
1079             dz22             = _mm_sub_ps(iz2,jz2);
1080
1081             /* Calculate squared distance and things based on it */
1082             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1083             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1084             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1085             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1086             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1087             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1088             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1089             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1090             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1091
1092             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1093             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1094             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1095             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1096             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1097             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1098             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1099             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1100             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1101
1102             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1103             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
1104             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
1105             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1106             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1107             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1108             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1109             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1110             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1111
1112             fjx0             = _mm_setzero_ps();
1113             fjy0             = _mm_setzero_ps();
1114             fjz0             = _mm_setzero_ps();
1115             fjx1             = _mm_setzero_ps();
1116             fjy1             = _mm_setzero_ps();
1117             fjz1             = _mm_setzero_ps();
1118             fjx2             = _mm_setzero_ps();
1119             fjy2             = _mm_setzero_ps();
1120             fjz2             = _mm_setzero_ps();
1121
1122             /**************************
1123              * CALCULATE INTERACTIONS *
1124              **************************/
1125
1126             /* REACTION-FIELD ELECTROSTATICS */
1127             felec            = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
1128
1129             /* LENNARD-JONES DISPERSION/REPULSION */
1130
1131             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1132             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1133
1134             fscal            = _mm_add_ps(felec,fvdw);
1135
1136              /* Update vectorial force */
1137             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1138             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1139             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1140
1141             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1142             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1143             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1144
1145             /**************************
1146              * CALCULATE INTERACTIONS *
1147              **************************/
1148
1149             /* REACTION-FIELD ELECTROSTATICS */
1150             felec            = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
1151
1152             fscal            = felec;
1153
1154              /* Update vectorial force */
1155             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1156             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1157             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1158
1159             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1160             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1161             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1162
1163             /**************************
1164              * CALCULATE INTERACTIONS *
1165              **************************/
1166
1167             /* REACTION-FIELD ELECTROSTATICS */
1168             felec            = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
1169
1170             fscal            = felec;
1171
1172              /* Update vectorial force */
1173             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1174             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1175             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1176
1177             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1178             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1179             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1180
1181             /**************************
1182              * CALCULATE INTERACTIONS *
1183              **************************/
1184
1185             /* REACTION-FIELD ELECTROSTATICS */
1186             felec            = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
1187
1188             fscal            = felec;
1189
1190              /* Update vectorial force */
1191             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1192             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1193             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1194
1195             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1196             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1197             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1198
1199             /**************************
1200              * CALCULATE INTERACTIONS *
1201              **************************/
1202
1203             /* REACTION-FIELD ELECTROSTATICS */
1204             felec            = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
1205
1206             fscal            = felec;
1207
1208              /* Update vectorial force */
1209             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1210             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1211             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1212
1213             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1214             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1215             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1216
1217             /**************************
1218              * CALCULATE INTERACTIONS *
1219              **************************/
1220
1221             /* REACTION-FIELD ELECTROSTATICS */
1222             felec            = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
1223
1224             fscal            = felec;
1225
1226              /* Update vectorial force */
1227             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1228             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1229             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1230
1231             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1232             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1233             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1234
1235             /**************************
1236              * CALCULATE INTERACTIONS *
1237              **************************/
1238
1239             /* REACTION-FIELD ELECTROSTATICS */
1240             felec            = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
1241
1242             fscal            = felec;
1243
1244              /* Update vectorial force */
1245             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1246             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1247             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1248
1249             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1250             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1251             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1252
1253             /**************************
1254              * CALCULATE INTERACTIONS *
1255              **************************/
1256
1257             /* REACTION-FIELD ELECTROSTATICS */
1258             felec            = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
1259
1260             fscal            = felec;
1261
1262              /* Update vectorial force */
1263             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1264             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1265             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1266
1267             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1268             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1269             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1270
1271             /**************************
1272              * CALCULATE INTERACTIONS *
1273              **************************/
1274
1275             /* REACTION-FIELD ELECTROSTATICS */
1276             felec            = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
1277
1278             fscal            = felec;
1279
1280              /* Update vectorial force */
1281             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1282             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1283             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1284
1285             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1286             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1287             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1288
1289             fjptrA             = f+j_coord_offsetA;
1290             fjptrB             = f+j_coord_offsetB;
1291             fjptrC             = f+j_coord_offsetC;
1292             fjptrD             = f+j_coord_offsetD;
1293
1294             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1295                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1296
1297             /* Inner loop uses 277 flops */
1298         }
1299
1300         if(jidx<j_index_end)
1301         {
1302
1303             /* Get j neighbor index, and coordinate index */
1304             jnrlistA         = jjnr[jidx];
1305             jnrlistB         = jjnr[jidx+1];
1306             jnrlistC         = jjnr[jidx+2];
1307             jnrlistD         = jjnr[jidx+3];
1308             /* Sign of each element will be negative for non-real atoms.
1309              * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1310              * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1311              */
1312             dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1313             jnrA       = (jnrlistA>=0) ? jnrlistA : 0;
1314             jnrB       = (jnrlistB>=0) ? jnrlistB : 0;
1315             jnrC       = (jnrlistC>=0) ? jnrlistC : 0;
1316             jnrD       = (jnrlistD>=0) ? jnrlistD : 0;
1317             j_coord_offsetA  = DIM*jnrA;
1318             j_coord_offsetB  = DIM*jnrB;
1319             j_coord_offsetC  = DIM*jnrC;
1320             j_coord_offsetD  = DIM*jnrD;
1321
1322             /* load j atom coordinates */
1323             gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1324                                               x+j_coord_offsetC,x+j_coord_offsetD,
1325                                               &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1326
1327             /* Calculate displacement vector */
1328             dx00             = _mm_sub_ps(ix0,jx0);
1329             dy00             = _mm_sub_ps(iy0,jy0);
1330             dz00             = _mm_sub_ps(iz0,jz0);
1331             dx01             = _mm_sub_ps(ix0,jx1);
1332             dy01             = _mm_sub_ps(iy0,jy1);
1333             dz01             = _mm_sub_ps(iz0,jz1);
1334             dx02             = _mm_sub_ps(ix0,jx2);
1335             dy02             = _mm_sub_ps(iy0,jy2);
1336             dz02             = _mm_sub_ps(iz0,jz2);
1337             dx10             = _mm_sub_ps(ix1,jx0);
1338             dy10             = _mm_sub_ps(iy1,jy0);
1339             dz10             = _mm_sub_ps(iz1,jz0);
1340             dx11             = _mm_sub_ps(ix1,jx1);
1341             dy11             = _mm_sub_ps(iy1,jy1);
1342             dz11             = _mm_sub_ps(iz1,jz1);
1343             dx12             = _mm_sub_ps(ix1,jx2);
1344             dy12             = _mm_sub_ps(iy1,jy2);
1345             dz12             = _mm_sub_ps(iz1,jz2);
1346             dx20             = _mm_sub_ps(ix2,jx0);
1347             dy20             = _mm_sub_ps(iy2,jy0);
1348             dz20             = _mm_sub_ps(iz2,jz0);
1349             dx21             = _mm_sub_ps(ix2,jx1);
1350             dy21             = _mm_sub_ps(iy2,jy1);
1351             dz21             = _mm_sub_ps(iz2,jz1);
1352             dx22             = _mm_sub_ps(ix2,jx2);
1353             dy22             = _mm_sub_ps(iy2,jy2);
1354             dz22             = _mm_sub_ps(iz2,jz2);
1355
1356             /* Calculate squared distance and things based on it */
1357             rsq00            = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1358             rsq01            = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1359             rsq02            = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1360             rsq10            = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1361             rsq11            = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1362             rsq12            = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1363             rsq20            = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1364             rsq21            = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1365             rsq22            = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1366
1367             rinv00           = gmx_mm_invsqrt_ps(rsq00);
1368             rinv01           = gmx_mm_invsqrt_ps(rsq01);
1369             rinv02           = gmx_mm_invsqrt_ps(rsq02);
1370             rinv10           = gmx_mm_invsqrt_ps(rsq10);
1371             rinv11           = gmx_mm_invsqrt_ps(rsq11);
1372             rinv12           = gmx_mm_invsqrt_ps(rsq12);
1373             rinv20           = gmx_mm_invsqrt_ps(rsq20);
1374             rinv21           = gmx_mm_invsqrt_ps(rsq21);
1375             rinv22           = gmx_mm_invsqrt_ps(rsq22);
1376
1377             rinvsq00         = _mm_mul_ps(rinv00,rinv00);
1378             rinvsq01         = _mm_mul_ps(rinv01,rinv01);
1379             rinvsq02         = _mm_mul_ps(rinv02,rinv02);
1380             rinvsq10         = _mm_mul_ps(rinv10,rinv10);
1381             rinvsq11         = _mm_mul_ps(rinv11,rinv11);
1382             rinvsq12         = _mm_mul_ps(rinv12,rinv12);
1383             rinvsq20         = _mm_mul_ps(rinv20,rinv20);
1384             rinvsq21         = _mm_mul_ps(rinv21,rinv21);
1385             rinvsq22         = _mm_mul_ps(rinv22,rinv22);
1386
1387             fjx0             = _mm_setzero_ps();
1388             fjy0             = _mm_setzero_ps();
1389             fjz0             = _mm_setzero_ps();
1390             fjx1             = _mm_setzero_ps();
1391             fjy1             = _mm_setzero_ps();
1392             fjz1             = _mm_setzero_ps();
1393             fjx2             = _mm_setzero_ps();
1394             fjy2             = _mm_setzero_ps();
1395             fjz2             = _mm_setzero_ps();
1396
1397             /**************************
1398              * CALCULATE INTERACTIONS *
1399              **************************/
1400
1401             /* REACTION-FIELD ELECTROSTATICS */
1402             felec            = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
1403
1404             /* LENNARD-JONES DISPERSION/REPULSION */
1405
1406             rinvsix          = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1407             fvdw             = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1408
1409             fscal            = _mm_add_ps(felec,fvdw);
1410
1411             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1412
1413              /* Update vectorial force */
1414             fix0             = _mm_macc_ps(dx00,fscal,fix0);
1415             fiy0             = _mm_macc_ps(dy00,fscal,fiy0);
1416             fiz0             = _mm_macc_ps(dz00,fscal,fiz0);
1417
1418             fjx0             = _mm_macc_ps(dx00,fscal,fjx0);
1419             fjy0             = _mm_macc_ps(dy00,fscal,fjy0);
1420             fjz0             = _mm_macc_ps(dz00,fscal,fjz0);
1421
1422             /**************************
1423              * CALCULATE INTERACTIONS *
1424              **************************/
1425
1426             /* REACTION-FIELD ELECTROSTATICS */
1427             felec            = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
1428
1429             fscal            = felec;
1430
1431             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1432
1433              /* Update vectorial force */
1434             fix0             = _mm_macc_ps(dx01,fscal,fix0);
1435             fiy0             = _mm_macc_ps(dy01,fscal,fiy0);
1436             fiz0             = _mm_macc_ps(dz01,fscal,fiz0);
1437
1438             fjx1             = _mm_macc_ps(dx01,fscal,fjx1);
1439             fjy1             = _mm_macc_ps(dy01,fscal,fjy1);
1440             fjz1             = _mm_macc_ps(dz01,fscal,fjz1);
1441
1442             /**************************
1443              * CALCULATE INTERACTIONS *
1444              **************************/
1445
1446             /* REACTION-FIELD ELECTROSTATICS */
1447             felec            = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
1448
1449             fscal            = felec;
1450
1451             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1452
1453              /* Update vectorial force */
1454             fix0             = _mm_macc_ps(dx02,fscal,fix0);
1455             fiy0             = _mm_macc_ps(dy02,fscal,fiy0);
1456             fiz0             = _mm_macc_ps(dz02,fscal,fiz0);
1457
1458             fjx2             = _mm_macc_ps(dx02,fscal,fjx2);
1459             fjy2             = _mm_macc_ps(dy02,fscal,fjy2);
1460             fjz2             = _mm_macc_ps(dz02,fscal,fjz2);
1461
1462             /**************************
1463              * CALCULATE INTERACTIONS *
1464              **************************/
1465
1466             /* REACTION-FIELD ELECTROSTATICS */
1467             felec            = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
1468
1469             fscal            = felec;
1470
1471             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1472
1473              /* Update vectorial force */
1474             fix1             = _mm_macc_ps(dx10,fscal,fix1);
1475             fiy1             = _mm_macc_ps(dy10,fscal,fiy1);
1476             fiz1             = _mm_macc_ps(dz10,fscal,fiz1);
1477
1478             fjx0             = _mm_macc_ps(dx10,fscal,fjx0);
1479             fjy0             = _mm_macc_ps(dy10,fscal,fjy0);
1480             fjz0             = _mm_macc_ps(dz10,fscal,fjz0);
1481
1482             /**************************
1483              * CALCULATE INTERACTIONS *
1484              **************************/
1485
1486             /* REACTION-FIELD ELECTROSTATICS */
1487             felec            = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
1488
1489             fscal            = felec;
1490
1491             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1492
1493              /* Update vectorial force */
1494             fix1             = _mm_macc_ps(dx11,fscal,fix1);
1495             fiy1             = _mm_macc_ps(dy11,fscal,fiy1);
1496             fiz1             = _mm_macc_ps(dz11,fscal,fiz1);
1497
1498             fjx1             = _mm_macc_ps(dx11,fscal,fjx1);
1499             fjy1             = _mm_macc_ps(dy11,fscal,fjy1);
1500             fjz1             = _mm_macc_ps(dz11,fscal,fjz1);
1501
1502             /**************************
1503              * CALCULATE INTERACTIONS *
1504              **************************/
1505
1506             /* REACTION-FIELD ELECTROSTATICS */
1507             felec            = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
1508
1509             fscal            = felec;
1510
1511             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1512
1513              /* Update vectorial force */
1514             fix1             = _mm_macc_ps(dx12,fscal,fix1);
1515             fiy1             = _mm_macc_ps(dy12,fscal,fiy1);
1516             fiz1             = _mm_macc_ps(dz12,fscal,fiz1);
1517
1518             fjx2             = _mm_macc_ps(dx12,fscal,fjx2);
1519             fjy2             = _mm_macc_ps(dy12,fscal,fjy2);
1520             fjz2             = _mm_macc_ps(dz12,fscal,fjz2);
1521
1522             /**************************
1523              * CALCULATE INTERACTIONS *
1524              **************************/
1525
1526             /* REACTION-FIELD ELECTROSTATICS */
1527             felec            = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
1528
1529             fscal            = felec;
1530
1531             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1532
1533              /* Update vectorial force */
1534             fix2             = _mm_macc_ps(dx20,fscal,fix2);
1535             fiy2             = _mm_macc_ps(dy20,fscal,fiy2);
1536             fiz2             = _mm_macc_ps(dz20,fscal,fiz2);
1537
1538             fjx0             = _mm_macc_ps(dx20,fscal,fjx0);
1539             fjy0             = _mm_macc_ps(dy20,fscal,fjy0);
1540             fjz0             = _mm_macc_ps(dz20,fscal,fjz0);
1541
1542             /**************************
1543              * CALCULATE INTERACTIONS *
1544              **************************/
1545
1546             /* REACTION-FIELD ELECTROSTATICS */
1547             felec            = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
1548
1549             fscal            = felec;
1550
1551             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1552
1553              /* Update vectorial force */
1554             fix2             = _mm_macc_ps(dx21,fscal,fix2);
1555             fiy2             = _mm_macc_ps(dy21,fscal,fiy2);
1556             fiz2             = _mm_macc_ps(dz21,fscal,fiz2);
1557
1558             fjx1             = _mm_macc_ps(dx21,fscal,fjx1);
1559             fjy1             = _mm_macc_ps(dy21,fscal,fjy1);
1560             fjz1             = _mm_macc_ps(dz21,fscal,fjz1);
1561
1562             /**************************
1563              * CALCULATE INTERACTIONS *
1564              **************************/
1565
1566             /* REACTION-FIELD ELECTROSTATICS */
1567             felec            = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
1568
1569             fscal            = felec;
1570
1571             fscal            = _mm_andnot_ps(dummy_mask,fscal);
1572
1573              /* Update vectorial force */
1574             fix2             = _mm_macc_ps(dx22,fscal,fix2);
1575             fiy2             = _mm_macc_ps(dy22,fscal,fiy2);
1576             fiz2             = _mm_macc_ps(dz22,fscal,fiz2);
1577
1578             fjx2             = _mm_macc_ps(dx22,fscal,fjx2);
1579             fjy2             = _mm_macc_ps(dy22,fscal,fjy2);
1580             fjz2             = _mm_macc_ps(dz22,fscal,fjz2);
1581
1582             fjptrA             = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1583             fjptrB             = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1584             fjptrC             = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1585             fjptrD             = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1586
1587             gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1588                                                    fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1589
1590             /* Inner loop uses 277 flops */
1591         }
1592
1593         /* End of innermost loop */
1594
1595         gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1596                                               f+i_coord_offset,fshift+i_shift_offset);
1597
1598         /* Increment number of inner iterations */
1599         inneriter                  += j_index_end - j_index_start;
1600
1601         /* Outer loop uses 18 flops */
1602     }
1603
1604     /* Increment number of outer iterations */
1605     outeriter        += nri;
1606
1607     /* Update outer/inner flops */
1608
1609     inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*277);
1610 }