2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015,2017,2018, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/gmxlib/nrnb.h"
47 #include "kernelutil_x86_avx_128_fma_single.h"
50 * Gromacs nonbonded kernel: nb_kernel_ElecCSTab_VdwCSTab_GeomW4P1_VF_avx_128_fma_single
51 * Electrostatics interaction: CubicSplineTable
52 * VdW interaction: CubicSplineTable
53 * Geometry: Water4-Particle
54 * Calculate force/pot: PotentialAndForce
57 nb_kernel_ElecCSTab_VdwCSTab_GeomW4P1_VF_avx_128_fma_single
58 (t_nblist * gmx_restrict nlist,
59 rvec * gmx_restrict xx,
60 rvec * gmx_restrict ff,
61 struct t_forcerec * gmx_restrict fr,
62 t_mdatoms * gmx_restrict mdatoms,
63 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
64 t_nrnb * gmx_restrict nrnb)
66 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
67 * just 0 for non-waters.
68 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
69 * jnr indices corresponding to data put in the four positions in the SIMD register.
71 int i_shift_offset,i_coord_offset,outeriter,inneriter;
72 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
73 int jnrA,jnrB,jnrC,jnrD;
74 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
75 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
76 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
78 real *shiftvec,*fshift,*x,*f;
79 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
81 __m128 fscal,rcutoff,rcutoff2,jidxall;
83 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
85 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
87 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
89 __m128 ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
90 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
91 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
92 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
93 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
94 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
95 __m128 dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
96 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
99 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
102 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
103 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
105 __m128i ifour = _mm_set1_epi32(4);
106 __m128 rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
108 __m128 dummy_mask,cutoff_mask;
109 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
110 __m128 one = _mm_set1_ps(1.0);
111 __m128 two = _mm_set1_ps(2.0);
117 jindex = nlist->jindex;
119 shiftidx = nlist->shift;
121 shiftvec = fr->shift_vec[0];
122 fshift = fr->fshift[0];
123 facel = _mm_set1_ps(fr->ic->epsfac);
124 charge = mdatoms->chargeA;
125 nvdwtype = fr->ntype;
127 vdwtype = mdatoms->typeA;
129 vftab = kernel_data->table_elec_vdw->data;
130 vftabscale = _mm_set1_ps(kernel_data->table_elec_vdw->scale);
132 /* Setup water-specific parameters */
133 inr = nlist->iinr[0];
134 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
135 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
136 iq3 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
137 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
139 /* Avoid stupid compiler warnings */
140 jnrA = jnrB = jnrC = jnrD = 0;
149 for(iidx=0;iidx<4*DIM;iidx++)
154 /* Start outer loop over neighborlists */
155 for(iidx=0; iidx<nri; iidx++)
157 /* Load shift vector for this list */
158 i_shift_offset = DIM*shiftidx[iidx];
160 /* Load limits for loop over neighbors */
161 j_index_start = jindex[iidx];
162 j_index_end = jindex[iidx+1];
164 /* Get outer coordinate index */
166 i_coord_offset = DIM*inr;
168 /* Load i particle coords and add shift vector */
169 gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
170 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
172 fix0 = _mm_setzero_ps();
173 fiy0 = _mm_setzero_ps();
174 fiz0 = _mm_setzero_ps();
175 fix1 = _mm_setzero_ps();
176 fiy1 = _mm_setzero_ps();
177 fiz1 = _mm_setzero_ps();
178 fix2 = _mm_setzero_ps();
179 fiy2 = _mm_setzero_ps();
180 fiz2 = _mm_setzero_ps();
181 fix3 = _mm_setzero_ps();
182 fiy3 = _mm_setzero_ps();
183 fiz3 = _mm_setzero_ps();
185 /* Reset potential sums */
186 velecsum = _mm_setzero_ps();
187 vvdwsum = _mm_setzero_ps();
189 /* Start inner kernel loop */
190 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
193 /* Get j neighbor index, and coordinate index */
198 j_coord_offsetA = DIM*jnrA;
199 j_coord_offsetB = DIM*jnrB;
200 j_coord_offsetC = DIM*jnrC;
201 j_coord_offsetD = DIM*jnrD;
203 /* load j atom coordinates */
204 gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
205 x+j_coord_offsetC,x+j_coord_offsetD,
208 /* Calculate displacement vector */
209 dx00 = _mm_sub_ps(ix0,jx0);
210 dy00 = _mm_sub_ps(iy0,jy0);
211 dz00 = _mm_sub_ps(iz0,jz0);
212 dx10 = _mm_sub_ps(ix1,jx0);
213 dy10 = _mm_sub_ps(iy1,jy0);
214 dz10 = _mm_sub_ps(iz1,jz0);
215 dx20 = _mm_sub_ps(ix2,jx0);
216 dy20 = _mm_sub_ps(iy2,jy0);
217 dz20 = _mm_sub_ps(iz2,jz0);
218 dx30 = _mm_sub_ps(ix3,jx0);
219 dy30 = _mm_sub_ps(iy3,jy0);
220 dz30 = _mm_sub_ps(iz3,jz0);
222 /* Calculate squared distance and things based on it */
223 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
224 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
225 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
226 rsq30 = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
228 rinv00 = avx128fma_invsqrt_f(rsq00);
229 rinv10 = avx128fma_invsqrt_f(rsq10);
230 rinv20 = avx128fma_invsqrt_f(rsq20);
231 rinv30 = avx128fma_invsqrt_f(rsq30);
233 /* Load parameters for j particles */
234 jq0 = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
235 charge+jnrC+0,charge+jnrD+0);
236 vdwjidx0A = 2*vdwtype[jnrA+0];
237 vdwjidx0B = 2*vdwtype[jnrB+0];
238 vdwjidx0C = 2*vdwtype[jnrC+0];
239 vdwjidx0D = 2*vdwtype[jnrD+0];
241 fjx0 = _mm_setzero_ps();
242 fjy0 = _mm_setzero_ps();
243 fjz0 = _mm_setzero_ps();
245 /**************************
246 * CALCULATE INTERACTIONS *
247 **************************/
249 r00 = _mm_mul_ps(rsq00,rinv00);
251 /* Compute parameters for interactions between i and j atoms */
252 gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
253 vdwparam+vdwioffset0+vdwjidx0B,
254 vdwparam+vdwioffset0+vdwjidx0C,
255 vdwparam+vdwioffset0+vdwjidx0D,
258 /* Calculate table index by multiplying r with table scale and truncate to integer */
259 rt = _mm_mul_ps(r00,vftabscale);
260 vfitab = _mm_cvttps_epi32(rt);
262 vfeps = _mm_frcz_ps(rt);
264 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
266 twovfeps = _mm_add_ps(vfeps,vfeps);
267 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
269 /* CUBIC SPLINE TABLE DISPERSION */
270 vfitab = _mm_add_epi32(vfitab,ifour);
271 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
272 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
273 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
274 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
275 _MM_TRANSPOSE4_PS(Y,F,G,H);
276 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
277 VV = _mm_macc_ps(vfeps,Fp,Y);
278 vvdw6 = _mm_mul_ps(c6_00,VV);
279 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
280 fvdw6 = _mm_mul_ps(c6_00,FF);
282 /* CUBIC SPLINE TABLE REPULSION */
283 vfitab = _mm_add_epi32(vfitab,ifour);
284 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
285 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
286 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
287 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
288 _MM_TRANSPOSE4_PS(Y,F,G,H);
289 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
290 VV = _mm_macc_ps(vfeps,Fp,Y);
291 vvdw12 = _mm_mul_ps(c12_00,VV);
292 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
293 fvdw12 = _mm_mul_ps(c12_00,FF);
294 vvdw = _mm_add_ps(vvdw12,vvdw6);
295 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
297 /* Update potential sum for this i atom from the interaction with this j atom. */
298 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
302 /* Update vectorial force */
303 fix0 = _mm_macc_ps(dx00,fscal,fix0);
304 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
305 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
307 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
308 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
309 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
311 /**************************
312 * CALCULATE INTERACTIONS *
313 **************************/
315 r10 = _mm_mul_ps(rsq10,rinv10);
317 /* Compute parameters for interactions between i and j atoms */
318 qq10 = _mm_mul_ps(iq1,jq0);
320 /* Calculate table index by multiplying r with table scale and truncate to integer */
321 rt = _mm_mul_ps(r10,vftabscale);
322 vfitab = _mm_cvttps_epi32(rt);
324 vfeps = _mm_frcz_ps(rt);
326 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
328 twovfeps = _mm_add_ps(vfeps,vfeps);
329 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
331 /* CUBIC SPLINE TABLE ELECTROSTATICS */
332 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
333 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
334 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
335 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
336 _MM_TRANSPOSE4_PS(Y,F,G,H);
337 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
338 VV = _mm_macc_ps(vfeps,Fp,Y);
339 velec = _mm_mul_ps(qq10,VV);
340 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
341 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
343 /* Update potential sum for this i atom from the interaction with this j atom. */
344 velecsum = _mm_add_ps(velecsum,velec);
348 /* Update vectorial force */
349 fix1 = _mm_macc_ps(dx10,fscal,fix1);
350 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
351 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
353 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
354 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
355 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
357 /**************************
358 * CALCULATE INTERACTIONS *
359 **************************/
361 r20 = _mm_mul_ps(rsq20,rinv20);
363 /* Compute parameters for interactions between i and j atoms */
364 qq20 = _mm_mul_ps(iq2,jq0);
366 /* Calculate table index by multiplying r with table scale and truncate to integer */
367 rt = _mm_mul_ps(r20,vftabscale);
368 vfitab = _mm_cvttps_epi32(rt);
370 vfeps = _mm_frcz_ps(rt);
372 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
374 twovfeps = _mm_add_ps(vfeps,vfeps);
375 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
377 /* CUBIC SPLINE TABLE ELECTROSTATICS */
378 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
379 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
380 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
381 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
382 _MM_TRANSPOSE4_PS(Y,F,G,H);
383 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
384 VV = _mm_macc_ps(vfeps,Fp,Y);
385 velec = _mm_mul_ps(qq20,VV);
386 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
387 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
389 /* Update potential sum for this i atom from the interaction with this j atom. */
390 velecsum = _mm_add_ps(velecsum,velec);
394 /* Update vectorial force */
395 fix2 = _mm_macc_ps(dx20,fscal,fix2);
396 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
397 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
399 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
400 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
401 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
403 /**************************
404 * CALCULATE INTERACTIONS *
405 **************************/
407 r30 = _mm_mul_ps(rsq30,rinv30);
409 /* Compute parameters for interactions between i and j atoms */
410 qq30 = _mm_mul_ps(iq3,jq0);
412 /* Calculate table index by multiplying r with table scale and truncate to integer */
413 rt = _mm_mul_ps(r30,vftabscale);
414 vfitab = _mm_cvttps_epi32(rt);
416 vfeps = _mm_frcz_ps(rt);
418 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
420 twovfeps = _mm_add_ps(vfeps,vfeps);
421 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
423 /* CUBIC SPLINE TABLE ELECTROSTATICS */
424 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
425 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
426 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
427 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
428 _MM_TRANSPOSE4_PS(Y,F,G,H);
429 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
430 VV = _mm_macc_ps(vfeps,Fp,Y);
431 velec = _mm_mul_ps(qq30,VV);
432 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
433 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq30,FF),_mm_mul_ps(vftabscale,rinv30)));
435 /* Update potential sum for this i atom from the interaction with this j atom. */
436 velecsum = _mm_add_ps(velecsum,velec);
440 /* Update vectorial force */
441 fix3 = _mm_macc_ps(dx30,fscal,fix3);
442 fiy3 = _mm_macc_ps(dy30,fscal,fiy3);
443 fiz3 = _mm_macc_ps(dz30,fscal,fiz3);
445 fjx0 = _mm_macc_ps(dx30,fscal,fjx0);
446 fjy0 = _mm_macc_ps(dy30,fscal,fjy0);
447 fjz0 = _mm_macc_ps(dz30,fscal,fjz0);
449 fjptrA = f+j_coord_offsetA;
450 fjptrB = f+j_coord_offsetB;
451 fjptrC = f+j_coord_offsetC;
452 fjptrD = f+j_coord_offsetD;
454 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
456 /* Inner loop uses 197 flops */
462 /* Get j neighbor index, and coordinate index */
463 jnrlistA = jjnr[jidx];
464 jnrlistB = jjnr[jidx+1];
465 jnrlistC = jjnr[jidx+2];
466 jnrlistD = jjnr[jidx+3];
467 /* Sign of each element will be negative for non-real atoms.
468 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
469 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
471 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
472 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
473 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
474 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
475 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
476 j_coord_offsetA = DIM*jnrA;
477 j_coord_offsetB = DIM*jnrB;
478 j_coord_offsetC = DIM*jnrC;
479 j_coord_offsetD = DIM*jnrD;
481 /* load j atom coordinates */
482 gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
483 x+j_coord_offsetC,x+j_coord_offsetD,
486 /* Calculate displacement vector */
487 dx00 = _mm_sub_ps(ix0,jx0);
488 dy00 = _mm_sub_ps(iy0,jy0);
489 dz00 = _mm_sub_ps(iz0,jz0);
490 dx10 = _mm_sub_ps(ix1,jx0);
491 dy10 = _mm_sub_ps(iy1,jy0);
492 dz10 = _mm_sub_ps(iz1,jz0);
493 dx20 = _mm_sub_ps(ix2,jx0);
494 dy20 = _mm_sub_ps(iy2,jy0);
495 dz20 = _mm_sub_ps(iz2,jz0);
496 dx30 = _mm_sub_ps(ix3,jx0);
497 dy30 = _mm_sub_ps(iy3,jy0);
498 dz30 = _mm_sub_ps(iz3,jz0);
500 /* Calculate squared distance and things based on it */
501 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
502 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
503 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
504 rsq30 = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
506 rinv00 = avx128fma_invsqrt_f(rsq00);
507 rinv10 = avx128fma_invsqrt_f(rsq10);
508 rinv20 = avx128fma_invsqrt_f(rsq20);
509 rinv30 = avx128fma_invsqrt_f(rsq30);
511 /* Load parameters for j particles */
512 jq0 = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
513 charge+jnrC+0,charge+jnrD+0);
514 vdwjidx0A = 2*vdwtype[jnrA+0];
515 vdwjidx0B = 2*vdwtype[jnrB+0];
516 vdwjidx0C = 2*vdwtype[jnrC+0];
517 vdwjidx0D = 2*vdwtype[jnrD+0];
519 fjx0 = _mm_setzero_ps();
520 fjy0 = _mm_setzero_ps();
521 fjz0 = _mm_setzero_ps();
523 /**************************
524 * CALCULATE INTERACTIONS *
525 **************************/
527 r00 = _mm_mul_ps(rsq00,rinv00);
528 r00 = _mm_andnot_ps(dummy_mask,r00);
530 /* Compute parameters for interactions between i and j atoms */
531 gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
532 vdwparam+vdwioffset0+vdwjidx0B,
533 vdwparam+vdwioffset0+vdwjidx0C,
534 vdwparam+vdwioffset0+vdwjidx0D,
537 /* Calculate table index by multiplying r with table scale and truncate to integer */
538 rt = _mm_mul_ps(r00,vftabscale);
539 vfitab = _mm_cvttps_epi32(rt);
541 vfeps = _mm_frcz_ps(rt);
543 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
545 twovfeps = _mm_add_ps(vfeps,vfeps);
546 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
548 /* CUBIC SPLINE TABLE DISPERSION */
549 vfitab = _mm_add_epi32(vfitab,ifour);
550 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
551 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
552 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
553 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
554 _MM_TRANSPOSE4_PS(Y,F,G,H);
555 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
556 VV = _mm_macc_ps(vfeps,Fp,Y);
557 vvdw6 = _mm_mul_ps(c6_00,VV);
558 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
559 fvdw6 = _mm_mul_ps(c6_00,FF);
561 /* CUBIC SPLINE TABLE REPULSION */
562 vfitab = _mm_add_epi32(vfitab,ifour);
563 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
564 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
565 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
566 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
567 _MM_TRANSPOSE4_PS(Y,F,G,H);
568 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
569 VV = _mm_macc_ps(vfeps,Fp,Y);
570 vvdw12 = _mm_mul_ps(c12_00,VV);
571 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
572 fvdw12 = _mm_mul_ps(c12_00,FF);
573 vvdw = _mm_add_ps(vvdw12,vvdw6);
574 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
576 /* Update potential sum for this i atom from the interaction with this j atom. */
577 vvdw = _mm_andnot_ps(dummy_mask,vvdw);
578 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
582 fscal = _mm_andnot_ps(dummy_mask,fscal);
584 /* Update vectorial force */
585 fix0 = _mm_macc_ps(dx00,fscal,fix0);
586 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
587 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
589 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
590 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
591 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
593 /**************************
594 * CALCULATE INTERACTIONS *
595 **************************/
597 r10 = _mm_mul_ps(rsq10,rinv10);
598 r10 = _mm_andnot_ps(dummy_mask,r10);
600 /* Compute parameters for interactions between i and j atoms */
601 qq10 = _mm_mul_ps(iq1,jq0);
603 /* Calculate table index by multiplying r with table scale and truncate to integer */
604 rt = _mm_mul_ps(r10,vftabscale);
605 vfitab = _mm_cvttps_epi32(rt);
607 vfeps = _mm_frcz_ps(rt);
609 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
611 twovfeps = _mm_add_ps(vfeps,vfeps);
612 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
614 /* CUBIC SPLINE TABLE ELECTROSTATICS */
615 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
616 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
617 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
618 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
619 _MM_TRANSPOSE4_PS(Y,F,G,H);
620 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
621 VV = _mm_macc_ps(vfeps,Fp,Y);
622 velec = _mm_mul_ps(qq10,VV);
623 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
624 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
626 /* Update potential sum for this i atom from the interaction with this j atom. */
627 velec = _mm_andnot_ps(dummy_mask,velec);
628 velecsum = _mm_add_ps(velecsum,velec);
632 fscal = _mm_andnot_ps(dummy_mask,fscal);
634 /* Update vectorial force */
635 fix1 = _mm_macc_ps(dx10,fscal,fix1);
636 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
637 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
639 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
640 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
641 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
643 /**************************
644 * CALCULATE INTERACTIONS *
645 **************************/
647 r20 = _mm_mul_ps(rsq20,rinv20);
648 r20 = _mm_andnot_ps(dummy_mask,r20);
650 /* Compute parameters for interactions between i and j atoms */
651 qq20 = _mm_mul_ps(iq2,jq0);
653 /* Calculate table index by multiplying r with table scale and truncate to integer */
654 rt = _mm_mul_ps(r20,vftabscale);
655 vfitab = _mm_cvttps_epi32(rt);
657 vfeps = _mm_frcz_ps(rt);
659 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
661 twovfeps = _mm_add_ps(vfeps,vfeps);
662 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
664 /* CUBIC SPLINE TABLE ELECTROSTATICS */
665 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
666 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
667 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
668 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
669 _MM_TRANSPOSE4_PS(Y,F,G,H);
670 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
671 VV = _mm_macc_ps(vfeps,Fp,Y);
672 velec = _mm_mul_ps(qq20,VV);
673 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
674 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
676 /* Update potential sum for this i atom from the interaction with this j atom. */
677 velec = _mm_andnot_ps(dummy_mask,velec);
678 velecsum = _mm_add_ps(velecsum,velec);
682 fscal = _mm_andnot_ps(dummy_mask,fscal);
684 /* Update vectorial force */
685 fix2 = _mm_macc_ps(dx20,fscal,fix2);
686 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
687 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
689 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
690 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
691 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
693 /**************************
694 * CALCULATE INTERACTIONS *
695 **************************/
697 r30 = _mm_mul_ps(rsq30,rinv30);
698 r30 = _mm_andnot_ps(dummy_mask,r30);
700 /* Compute parameters for interactions between i and j atoms */
701 qq30 = _mm_mul_ps(iq3,jq0);
703 /* Calculate table index by multiplying r with table scale and truncate to integer */
704 rt = _mm_mul_ps(r30,vftabscale);
705 vfitab = _mm_cvttps_epi32(rt);
707 vfeps = _mm_frcz_ps(rt);
709 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
711 twovfeps = _mm_add_ps(vfeps,vfeps);
712 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
714 /* CUBIC SPLINE TABLE ELECTROSTATICS */
715 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
716 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
717 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
718 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
719 _MM_TRANSPOSE4_PS(Y,F,G,H);
720 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
721 VV = _mm_macc_ps(vfeps,Fp,Y);
722 velec = _mm_mul_ps(qq30,VV);
723 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
724 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq30,FF),_mm_mul_ps(vftabscale,rinv30)));
726 /* Update potential sum for this i atom from the interaction with this j atom. */
727 velec = _mm_andnot_ps(dummy_mask,velec);
728 velecsum = _mm_add_ps(velecsum,velec);
732 fscal = _mm_andnot_ps(dummy_mask,fscal);
734 /* Update vectorial force */
735 fix3 = _mm_macc_ps(dx30,fscal,fix3);
736 fiy3 = _mm_macc_ps(dy30,fscal,fiy3);
737 fiz3 = _mm_macc_ps(dz30,fscal,fiz3);
739 fjx0 = _mm_macc_ps(dx30,fscal,fjx0);
740 fjy0 = _mm_macc_ps(dy30,fscal,fjy0);
741 fjz0 = _mm_macc_ps(dz30,fscal,fjz0);
743 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
744 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
745 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
746 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
748 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
750 /* Inner loop uses 201 flops */
753 /* End of innermost loop */
755 gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
756 f+i_coord_offset,fshift+i_shift_offset);
759 /* Update potential energies */
760 gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
761 gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
763 /* Increment number of inner iterations */
764 inneriter += j_index_end - j_index_start;
766 /* Outer loop uses 26 flops */
769 /* Increment number of outer iterations */
772 /* Update outer/inner flops */
774 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_VF,outeriter*26 + inneriter*201);
777 * Gromacs nonbonded kernel: nb_kernel_ElecCSTab_VdwCSTab_GeomW4P1_F_avx_128_fma_single
778 * Electrostatics interaction: CubicSplineTable
779 * VdW interaction: CubicSplineTable
780 * Geometry: Water4-Particle
781 * Calculate force/pot: Force
784 nb_kernel_ElecCSTab_VdwCSTab_GeomW4P1_F_avx_128_fma_single
785 (t_nblist * gmx_restrict nlist,
786 rvec * gmx_restrict xx,
787 rvec * gmx_restrict ff,
788 struct t_forcerec * gmx_restrict fr,
789 t_mdatoms * gmx_restrict mdatoms,
790 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
791 t_nrnb * gmx_restrict nrnb)
793 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
794 * just 0 for non-waters.
795 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
796 * jnr indices corresponding to data put in the four positions in the SIMD register.
798 int i_shift_offset,i_coord_offset,outeriter,inneriter;
799 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
800 int jnrA,jnrB,jnrC,jnrD;
801 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
802 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
803 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
805 real *shiftvec,*fshift,*x,*f;
806 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
808 __m128 fscal,rcutoff,rcutoff2,jidxall;
810 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
812 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
814 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
816 __m128 ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
817 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
818 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
819 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
820 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
821 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
822 __m128 dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
823 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
826 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
829 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
830 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
832 __m128i ifour = _mm_set1_epi32(4);
833 __m128 rt,vfeps,twovfeps,vftabscale,Y,F,G,H,Fp,VV,FF;
835 __m128 dummy_mask,cutoff_mask;
836 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
837 __m128 one = _mm_set1_ps(1.0);
838 __m128 two = _mm_set1_ps(2.0);
844 jindex = nlist->jindex;
846 shiftidx = nlist->shift;
848 shiftvec = fr->shift_vec[0];
849 fshift = fr->fshift[0];
850 facel = _mm_set1_ps(fr->ic->epsfac);
851 charge = mdatoms->chargeA;
852 nvdwtype = fr->ntype;
854 vdwtype = mdatoms->typeA;
856 vftab = kernel_data->table_elec_vdw->data;
857 vftabscale = _mm_set1_ps(kernel_data->table_elec_vdw->scale);
859 /* Setup water-specific parameters */
860 inr = nlist->iinr[0];
861 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
862 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
863 iq3 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+3]));
864 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
866 /* Avoid stupid compiler warnings */
867 jnrA = jnrB = jnrC = jnrD = 0;
876 for(iidx=0;iidx<4*DIM;iidx++)
881 /* Start outer loop over neighborlists */
882 for(iidx=0; iidx<nri; iidx++)
884 /* Load shift vector for this list */
885 i_shift_offset = DIM*shiftidx[iidx];
887 /* Load limits for loop over neighbors */
888 j_index_start = jindex[iidx];
889 j_index_end = jindex[iidx+1];
891 /* Get outer coordinate index */
893 i_coord_offset = DIM*inr;
895 /* Load i particle coords and add shift vector */
896 gmx_mm_load_shift_and_4rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
897 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
899 fix0 = _mm_setzero_ps();
900 fiy0 = _mm_setzero_ps();
901 fiz0 = _mm_setzero_ps();
902 fix1 = _mm_setzero_ps();
903 fiy1 = _mm_setzero_ps();
904 fiz1 = _mm_setzero_ps();
905 fix2 = _mm_setzero_ps();
906 fiy2 = _mm_setzero_ps();
907 fiz2 = _mm_setzero_ps();
908 fix3 = _mm_setzero_ps();
909 fiy3 = _mm_setzero_ps();
910 fiz3 = _mm_setzero_ps();
912 /* Start inner kernel loop */
913 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
916 /* Get j neighbor index, and coordinate index */
921 j_coord_offsetA = DIM*jnrA;
922 j_coord_offsetB = DIM*jnrB;
923 j_coord_offsetC = DIM*jnrC;
924 j_coord_offsetD = DIM*jnrD;
926 /* load j atom coordinates */
927 gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
928 x+j_coord_offsetC,x+j_coord_offsetD,
931 /* Calculate displacement vector */
932 dx00 = _mm_sub_ps(ix0,jx0);
933 dy00 = _mm_sub_ps(iy0,jy0);
934 dz00 = _mm_sub_ps(iz0,jz0);
935 dx10 = _mm_sub_ps(ix1,jx0);
936 dy10 = _mm_sub_ps(iy1,jy0);
937 dz10 = _mm_sub_ps(iz1,jz0);
938 dx20 = _mm_sub_ps(ix2,jx0);
939 dy20 = _mm_sub_ps(iy2,jy0);
940 dz20 = _mm_sub_ps(iz2,jz0);
941 dx30 = _mm_sub_ps(ix3,jx0);
942 dy30 = _mm_sub_ps(iy3,jy0);
943 dz30 = _mm_sub_ps(iz3,jz0);
945 /* Calculate squared distance and things based on it */
946 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
947 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
948 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
949 rsq30 = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
951 rinv00 = avx128fma_invsqrt_f(rsq00);
952 rinv10 = avx128fma_invsqrt_f(rsq10);
953 rinv20 = avx128fma_invsqrt_f(rsq20);
954 rinv30 = avx128fma_invsqrt_f(rsq30);
956 /* Load parameters for j particles */
957 jq0 = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
958 charge+jnrC+0,charge+jnrD+0);
959 vdwjidx0A = 2*vdwtype[jnrA+0];
960 vdwjidx0B = 2*vdwtype[jnrB+0];
961 vdwjidx0C = 2*vdwtype[jnrC+0];
962 vdwjidx0D = 2*vdwtype[jnrD+0];
964 fjx0 = _mm_setzero_ps();
965 fjy0 = _mm_setzero_ps();
966 fjz0 = _mm_setzero_ps();
968 /**************************
969 * CALCULATE INTERACTIONS *
970 **************************/
972 r00 = _mm_mul_ps(rsq00,rinv00);
974 /* Compute parameters for interactions between i and j atoms */
975 gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
976 vdwparam+vdwioffset0+vdwjidx0B,
977 vdwparam+vdwioffset0+vdwjidx0C,
978 vdwparam+vdwioffset0+vdwjidx0D,
981 /* Calculate table index by multiplying r with table scale and truncate to integer */
982 rt = _mm_mul_ps(r00,vftabscale);
983 vfitab = _mm_cvttps_epi32(rt);
985 vfeps = _mm_frcz_ps(rt);
987 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
989 twovfeps = _mm_add_ps(vfeps,vfeps);
990 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
992 /* CUBIC SPLINE TABLE DISPERSION */
993 vfitab = _mm_add_epi32(vfitab,ifour);
994 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
995 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
996 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
997 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
998 _MM_TRANSPOSE4_PS(Y,F,G,H);
999 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1000 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1001 fvdw6 = _mm_mul_ps(c6_00,FF);
1003 /* CUBIC SPLINE TABLE REPULSION */
1004 vfitab = _mm_add_epi32(vfitab,ifour);
1005 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1006 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1007 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1008 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1009 _MM_TRANSPOSE4_PS(Y,F,G,H);
1010 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1011 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1012 fvdw12 = _mm_mul_ps(c12_00,FF);
1013 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
1017 /* Update vectorial force */
1018 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1019 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1020 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1022 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1023 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1024 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1026 /**************************
1027 * CALCULATE INTERACTIONS *
1028 **************************/
1030 r10 = _mm_mul_ps(rsq10,rinv10);
1032 /* Compute parameters for interactions between i and j atoms */
1033 qq10 = _mm_mul_ps(iq1,jq0);
1035 /* Calculate table index by multiplying r with table scale and truncate to integer */
1036 rt = _mm_mul_ps(r10,vftabscale);
1037 vfitab = _mm_cvttps_epi32(rt);
1039 vfeps = _mm_frcz_ps(rt);
1041 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1043 twovfeps = _mm_add_ps(vfeps,vfeps);
1044 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1046 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1047 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1048 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1049 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1050 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1051 _MM_TRANSPOSE4_PS(Y,F,G,H);
1052 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1053 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1054 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
1058 /* Update vectorial force */
1059 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1060 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1061 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1063 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1064 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1065 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1067 /**************************
1068 * CALCULATE INTERACTIONS *
1069 **************************/
1071 r20 = _mm_mul_ps(rsq20,rinv20);
1073 /* Compute parameters for interactions between i and j atoms */
1074 qq20 = _mm_mul_ps(iq2,jq0);
1076 /* Calculate table index by multiplying r with table scale and truncate to integer */
1077 rt = _mm_mul_ps(r20,vftabscale);
1078 vfitab = _mm_cvttps_epi32(rt);
1080 vfeps = _mm_frcz_ps(rt);
1082 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1084 twovfeps = _mm_add_ps(vfeps,vfeps);
1085 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1087 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1088 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1089 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1090 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1091 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1092 _MM_TRANSPOSE4_PS(Y,F,G,H);
1093 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1094 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1095 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1099 /* Update vectorial force */
1100 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1101 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1102 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1104 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1105 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1106 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1108 /**************************
1109 * CALCULATE INTERACTIONS *
1110 **************************/
1112 r30 = _mm_mul_ps(rsq30,rinv30);
1114 /* Compute parameters for interactions between i and j atoms */
1115 qq30 = _mm_mul_ps(iq3,jq0);
1117 /* Calculate table index by multiplying r with table scale and truncate to integer */
1118 rt = _mm_mul_ps(r30,vftabscale);
1119 vfitab = _mm_cvttps_epi32(rt);
1121 vfeps = _mm_frcz_ps(rt);
1123 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1125 twovfeps = _mm_add_ps(vfeps,vfeps);
1126 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1128 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1129 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1130 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1131 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1132 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1133 _MM_TRANSPOSE4_PS(Y,F,G,H);
1134 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1135 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1136 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq30,FF),_mm_mul_ps(vftabscale,rinv30)));
1140 /* Update vectorial force */
1141 fix3 = _mm_macc_ps(dx30,fscal,fix3);
1142 fiy3 = _mm_macc_ps(dy30,fscal,fiy3);
1143 fiz3 = _mm_macc_ps(dz30,fscal,fiz3);
1145 fjx0 = _mm_macc_ps(dx30,fscal,fjx0);
1146 fjy0 = _mm_macc_ps(dy30,fscal,fjy0);
1147 fjz0 = _mm_macc_ps(dz30,fscal,fjz0);
1149 fjptrA = f+j_coord_offsetA;
1150 fjptrB = f+j_coord_offsetB;
1151 fjptrC = f+j_coord_offsetC;
1152 fjptrD = f+j_coord_offsetD;
1154 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1156 /* Inner loop uses 177 flops */
1159 if(jidx<j_index_end)
1162 /* Get j neighbor index, and coordinate index */
1163 jnrlistA = jjnr[jidx];
1164 jnrlistB = jjnr[jidx+1];
1165 jnrlistC = jjnr[jidx+2];
1166 jnrlistD = jjnr[jidx+3];
1167 /* Sign of each element will be negative for non-real atoms.
1168 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1169 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1171 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1172 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1173 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1174 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1175 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1176 j_coord_offsetA = DIM*jnrA;
1177 j_coord_offsetB = DIM*jnrB;
1178 j_coord_offsetC = DIM*jnrC;
1179 j_coord_offsetD = DIM*jnrD;
1181 /* load j atom coordinates */
1182 gmx_mm_load_1rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1183 x+j_coord_offsetC,x+j_coord_offsetD,
1186 /* Calculate displacement vector */
1187 dx00 = _mm_sub_ps(ix0,jx0);
1188 dy00 = _mm_sub_ps(iy0,jy0);
1189 dz00 = _mm_sub_ps(iz0,jz0);
1190 dx10 = _mm_sub_ps(ix1,jx0);
1191 dy10 = _mm_sub_ps(iy1,jy0);
1192 dz10 = _mm_sub_ps(iz1,jz0);
1193 dx20 = _mm_sub_ps(ix2,jx0);
1194 dy20 = _mm_sub_ps(iy2,jy0);
1195 dz20 = _mm_sub_ps(iz2,jz0);
1196 dx30 = _mm_sub_ps(ix3,jx0);
1197 dy30 = _mm_sub_ps(iy3,jy0);
1198 dz30 = _mm_sub_ps(iz3,jz0);
1200 /* Calculate squared distance and things based on it */
1201 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1202 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1203 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1204 rsq30 = gmx_mm_calc_rsq_ps(dx30,dy30,dz30);
1206 rinv00 = avx128fma_invsqrt_f(rsq00);
1207 rinv10 = avx128fma_invsqrt_f(rsq10);
1208 rinv20 = avx128fma_invsqrt_f(rsq20);
1209 rinv30 = avx128fma_invsqrt_f(rsq30);
1211 /* Load parameters for j particles */
1212 jq0 = gmx_mm_load_4real_swizzle_ps(charge+jnrA+0,charge+jnrB+0,
1213 charge+jnrC+0,charge+jnrD+0);
1214 vdwjidx0A = 2*vdwtype[jnrA+0];
1215 vdwjidx0B = 2*vdwtype[jnrB+0];
1216 vdwjidx0C = 2*vdwtype[jnrC+0];
1217 vdwjidx0D = 2*vdwtype[jnrD+0];
1219 fjx0 = _mm_setzero_ps();
1220 fjy0 = _mm_setzero_ps();
1221 fjz0 = _mm_setzero_ps();
1223 /**************************
1224 * CALCULATE INTERACTIONS *
1225 **************************/
1227 r00 = _mm_mul_ps(rsq00,rinv00);
1228 r00 = _mm_andnot_ps(dummy_mask,r00);
1230 /* Compute parameters for interactions between i and j atoms */
1231 gmx_mm_load_4pair_swizzle_ps(vdwparam+vdwioffset0+vdwjidx0A,
1232 vdwparam+vdwioffset0+vdwjidx0B,
1233 vdwparam+vdwioffset0+vdwjidx0C,
1234 vdwparam+vdwioffset0+vdwjidx0D,
1237 /* Calculate table index by multiplying r with table scale and truncate to integer */
1238 rt = _mm_mul_ps(r00,vftabscale);
1239 vfitab = _mm_cvttps_epi32(rt);
1241 vfeps = _mm_frcz_ps(rt);
1243 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1245 twovfeps = _mm_add_ps(vfeps,vfeps);
1246 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1248 /* CUBIC SPLINE TABLE DISPERSION */
1249 vfitab = _mm_add_epi32(vfitab,ifour);
1250 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1251 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1252 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1253 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1254 _MM_TRANSPOSE4_PS(Y,F,G,H);
1255 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1256 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1257 fvdw6 = _mm_mul_ps(c6_00,FF);
1259 /* CUBIC SPLINE TABLE REPULSION */
1260 vfitab = _mm_add_epi32(vfitab,ifour);
1261 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1262 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1263 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1264 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1265 _MM_TRANSPOSE4_PS(Y,F,G,H);
1266 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1267 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1268 fvdw12 = _mm_mul_ps(c12_00,FF);
1269 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
1273 fscal = _mm_andnot_ps(dummy_mask,fscal);
1275 /* Update vectorial force */
1276 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1277 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1278 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1280 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1281 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1282 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1284 /**************************
1285 * CALCULATE INTERACTIONS *
1286 **************************/
1288 r10 = _mm_mul_ps(rsq10,rinv10);
1289 r10 = _mm_andnot_ps(dummy_mask,r10);
1291 /* Compute parameters for interactions between i and j atoms */
1292 qq10 = _mm_mul_ps(iq1,jq0);
1294 /* Calculate table index by multiplying r with table scale and truncate to integer */
1295 rt = _mm_mul_ps(r10,vftabscale);
1296 vfitab = _mm_cvttps_epi32(rt);
1298 vfeps = _mm_frcz_ps(rt);
1300 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1302 twovfeps = _mm_add_ps(vfeps,vfeps);
1303 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1305 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1306 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1307 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1308 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1309 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1310 _MM_TRANSPOSE4_PS(Y,F,G,H);
1311 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1312 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1313 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq10,FF),_mm_mul_ps(vftabscale,rinv10)));
1317 fscal = _mm_andnot_ps(dummy_mask,fscal);
1319 /* Update vectorial force */
1320 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1321 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1322 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1324 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1325 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1326 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1328 /**************************
1329 * CALCULATE INTERACTIONS *
1330 **************************/
1332 r20 = _mm_mul_ps(rsq20,rinv20);
1333 r20 = _mm_andnot_ps(dummy_mask,r20);
1335 /* Compute parameters for interactions between i and j atoms */
1336 qq20 = _mm_mul_ps(iq2,jq0);
1338 /* Calculate table index by multiplying r with table scale and truncate to integer */
1339 rt = _mm_mul_ps(r20,vftabscale);
1340 vfitab = _mm_cvttps_epi32(rt);
1342 vfeps = _mm_frcz_ps(rt);
1344 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1346 twovfeps = _mm_add_ps(vfeps,vfeps);
1347 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1349 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1350 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1351 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1352 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1353 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1354 _MM_TRANSPOSE4_PS(Y,F,G,H);
1355 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1356 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1357 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq20,FF),_mm_mul_ps(vftabscale,rinv20)));
1361 fscal = _mm_andnot_ps(dummy_mask,fscal);
1363 /* Update vectorial force */
1364 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1365 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1366 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1368 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1369 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1370 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1372 /**************************
1373 * CALCULATE INTERACTIONS *
1374 **************************/
1376 r30 = _mm_mul_ps(rsq30,rinv30);
1377 r30 = _mm_andnot_ps(dummy_mask,r30);
1379 /* Compute parameters for interactions between i and j atoms */
1380 qq30 = _mm_mul_ps(iq3,jq0);
1382 /* Calculate table index by multiplying r with table scale and truncate to integer */
1383 rt = _mm_mul_ps(r30,vftabscale);
1384 vfitab = _mm_cvttps_epi32(rt);
1386 vfeps = _mm_frcz_ps(rt);
1388 vfeps = _mm_sub_ps(rt,_mm_round_ps(rt, _MM_FROUND_FLOOR));
1390 twovfeps = _mm_add_ps(vfeps,vfeps);
1391 vfitab = _mm_slli_epi32(_mm_add_epi32(vfitab,_mm_slli_epi32(vfitab,1)),2);
1393 /* CUBIC SPLINE TABLE ELECTROSTATICS */
1394 Y = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,0) );
1395 F = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,1) );
1396 G = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,2) );
1397 H = _mm_load_ps( vftab + _mm_extract_epi32(vfitab,3) );
1398 _MM_TRANSPOSE4_PS(Y,F,G,H);
1399 Fp = _mm_macc_ps(vfeps,_mm_macc_ps(H,vfeps,G),F);
1400 FF = _mm_macc_ps(vfeps,_mm_macc_ps(twovfeps,H,G),Fp);
1401 felec = _mm_xor_ps(signbit,_mm_mul_ps(_mm_mul_ps(qq30,FF),_mm_mul_ps(vftabscale,rinv30)));
1405 fscal = _mm_andnot_ps(dummy_mask,fscal);
1407 /* Update vectorial force */
1408 fix3 = _mm_macc_ps(dx30,fscal,fix3);
1409 fiy3 = _mm_macc_ps(dy30,fscal,fiy3);
1410 fiz3 = _mm_macc_ps(dz30,fscal,fiz3);
1412 fjx0 = _mm_macc_ps(dx30,fscal,fjx0);
1413 fjy0 = _mm_macc_ps(dy30,fscal,fjy0);
1414 fjz0 = _mm_macc_ps(dz30,fscal,fjz0);
1416 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1417 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1418 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1419 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1421 gmx_mm_decrement_1rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1423 /* Inner loop uses 181 flops */
1426 /* End of innermost loop */
1428 gmx_mm_update_iforce_4atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1429 f+i_coord_offset,fshift+i_shift_offset);
1431 /* Increment number of inner iterations */
1432 inneriter += j_index_end - j_index_start;
1434 /* Outer loop uses 24 flops */
1437 /* Increment number of outer iterations */
1440 /* Update outer/inner flops */
1442 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_F,outeriter*24 + inneriter*181);