2 * Note: this file was generated by the Gromacs avx_128_fma_double kernel generator.
4 * This source code is part of
8 * Copyright (c) 2001-2012, The GROMACS Development Team
10 * Gromacs is a library for molecular simulation and trajectory analysis,
11 * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
12 * a full list of developers and information, check out http://www.gromacs.org
14 * This program is free software; you can redistribute it and/or modify it under
15 * the terms of the GNU Lesser General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option) any
19 * To help fund GROMACS development, we humbly ask that you cite
20 * the papers people have written on it - you can find them on the website.
28 #include "../nb_kernel.h"
29 #include "types/simple.h"
33 #include "gmx_math_x86_avx_128_fma_double.h"
34 #include "kernelutil_x86_avx_128_fma_double.h"
37 * Gromacs nonbonded kernel: nb_kernel_ElecEwSh_VdwNone_GeomW4P1_VF_avx_128_fma_double
38 * Electrostatics interaction: Ewald
39 * VdW interaction: None
40 * Geometry: Water4-Particle
41 * Calculate force/pot: PotentialAndForce
44 nb_kernel_ElecEwSh_VdwNone_GeomW4P1_VF_avx_128_fma_double
45 (t_nblist * gmx_restrict nlist,
46 rvec * gmx_restrict xx,
47 rvec * gmx_restrict ff,
48 t_forcerec * gmx_restrict fr,
49 t_mdatoms * gmx_restrict mdatoms,
50 nb_kernel_data_t * gmx_restrict kernel_data,
51 t_nrnb * gmx_restrict nrnb)
53 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
54 * just 0 for non-waters.
55 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
56 * jnr indices corresponding to data put in the four positions in the SIMD register.
58 int i_shift_offset,i_coord_offset,outeriter,inneriter;
59 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
61 int j_coord_offsetA,j_coord_offsetB;
62 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
64 real *shiftvec,*fshift,*x,*f;
65 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
67 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
69 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
71 __m128d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
72 int vdwjidx0A,vdwjidx0B;
73 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
74 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
75 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
76 __m128d dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
77 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
80 __m128d ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
82 __m128d dummy_mask,cutoff_mask;
83 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
84 __m128d one = _mm_set1_pd(1.0);
85 __m128d two = _mm_set1_pd(2.0);
91 jindex = nlist->jindex;
93 shiftidx = nlist->shift;
95 shiftvec = fr->shift_vec[0];
96 fshift = fr->fshift[0];
97 facel = _mm_set1_pd(fr->epsfac);
98 charge = mdatoms->chargeA;
100 sh_ewald = _mm_set1_pd(fr->ic->sh_ewald);
101 ewtab = fr->ic->tabq_coul_FDV0;
102 ewtabscale = _mm_set1_pd(fr->ic->tabq_scale);
103 ewtabhalfspace = _mm_set1_pd(0.5/fr->ic->tabq_scale);
105 /* Setup water-specific parameters */
106 inr = nlist->iinr[0];
107 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
108 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
109 iq3 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
111 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
112 rcutoff_scalar = fr->rcoulomb;
113 rcutoff = _mm_set1_pd(rcutoff_scalar);
114 rcutoff2 = _mm_mul_pd(rcutoff,rcutoff);
116 /* Avoid stupid compiler warnings */
124 /* Start outer loop over neighborlists */
125 for(iidx=0; iidx<nri; iidx++)
127 /* Load shift vector for this list */
128 i_shift_offset = DIM*shiftidx[iidx];
130 /* Load limits for loop over neighbors */
131 j_index_start = jindex[iidx];
132 j_index_end = jindex[iidx+1];
134 /* Get outer coordinate index */
136 i_coord_offset = DIM*inr;
138 /* Load i particle coords and add shift vector */
139 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset+DIM,
140 &ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
142 fix1 = _mm_setzero_pd();
143 fiy1 = _mm_setzero_pd();
144 fiz1 = _mm_setzero_pd();
145 fix2 = _mm_setzero_pd();
146 fiy2 = _mm_setzero_pd();
147 fiz2 = _mm_setzero_pd();
148 fix3 = _mm_setzero_pd();
149 fiy3 = _mm_setzero_pd();
150 fiz3 = _mm_setzero_pd();
152 /* Reset potential sums */
153 velecsum = _mm_setzero_pd();
155 /* Start inner kernel loop */
156 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
159 /* Get j neighbor index, and coordinate index */
162 j_coord_offsetA = DIM*jnrA;
163 j_coord_offsetB = DIM*jnrB;
165 /* load j atom coordinates */
166 gmx_mm_load_1rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
169 /* Calculate displacement vector */
170 dx10 = _mm_sub_pd(ix1,jx0);
171 dy10 = _mm_sub_pd(iy1,jy0);
172 dz10 = _mm_sub_pd(iz1,jz0);
173 dx20 = _mm_sub_pd(ix2,jx0);
174 dy20 = _mm_sub_pd(iy2,jy0);
175 dz20 = _mm_sub_pd(iz2,jz0);
176 dx30 = _mm_sub_pd(ix3,jx0);
177 dy30 = _mm_sub_pd(iy3,jy0);
178 dz30 = _mm_sub_pd(iz3,jz0);
180 /* Calculate squared distance and things based on it */
181 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
182 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
183 rsq30 = gmx_mm_calc_rsq_pd(dx30,dy30,dz30);
185 rinv10 = gmx_mm_invsqrt_pd(rsq10);
186 rinv20 = gmx_mm_invsqrt_pd(rsq20);
187 rinv30 = gmx_mm_invsqrt_pd(rsq30);
189 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
190 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
191 rinvsq30 = _mm_mul_pd(rinv30,rinv30);
193 /* Load parameters for j particles */
194 jq0 = gmx_mm_load_2real_swizzle_pd(charge+jnrA+0,charge+jnrB+0);
196 fjx0 = _mm_setzero_pd();
197 fjy0 = _mm_setzero_pd();
198 fjz0 = _mm_setzero_pd();
200 /**************************
201 * CALCULATE INTERACTIONS *
202 **************************/
204 if (gmx_mm_any_lt(rsq10,rcutoff2))
207 r10 = _mm_mul_pd(rsq10,rinv10);
209 /* Compute parameters for interactions between i and j atoms */
210 qq10 = _mm_mul_pd(iq1,jq0);
212 /* EWALD ELECTROSTATICS */
214 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
215 ewrt = _mm_mul_pd(r10,ewtabscale);
216 ewitab = _mm_cvttpd_epi32(ewrt);
218 eweps = _mm_frcz_pd(ewrt);
220 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
222 twoeweps = _mm_add_pd(eweps,eweps);
223 ewitab = _mm_slli_epi32(ewitab,2);
224 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
225 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
226 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
227 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
228 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
229 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
230 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
231 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
232 velec = _mm_mul_pd(qq10,_mm_sub_pd(_mm_sub_pd(rinv10,sh_ewald),velec));
233 felec = _mm_mul_pd(_mm_mul_pd(qq10,rinv10),_mm_sub_pd(rinvsq10,felec));
235 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
237 /* Update potential sum for this i atom from the interaction with this j atom. */
238 velec = _mm_and_pd(velec,cutoff_mask);
239 velecsum = _mm_add_pd(velecsum,velec);
243 fscal = _mm_and_pd(fscal,cutoff_mask);
245 /* Update vectorial force */
246 fix1 = _mm_macc_pd(dx10,fscal,fix1);
247 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
248 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
250 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
251 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
252 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
256 /**************************
257 * CALCULATE INTERACTIONS *
258 **************************/
260 if (gmx_mm_any_lt(rsq20,rcutoff2))
263 r20 = _mm_mul_pd(rsq20,rinv20);
265 /* Compute parameters for interactions between i and j atoms */
266 qq20 = _mm_mul_pd(iq2,jq0);
268 /* EWALD ELECTROSTATICS */
270 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
271 ewrt = _mm_mul_pd(r20,ewtabscale);
272 ewitab = _mm_cvttpd_epi32(ewrt);
274 eweps = _mm_frcz_pd(ewrt);
276 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
278 twoeweps = _mm_add_pd(eweps,eweps);
279 ewitab = _mm_slli_epi32(ewitab,2);
280 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
281 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
282 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
283 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
284 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
285 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
286 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
287 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
288 velec = _mm_mul_pd(qq20,_mm_sub_pd(_mm_sub_pd(rinv20,sh_ewald),velec));
289 felec = _mm_mul_pd(_mm_mul_pd(qq20,rinv20),_mm_sub_pd(rinvsq20,felec));
291 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
293 /* Update potential sum for this i atom from the interaction with this j atom. */
294 velec = _mm_and_pd(velec,cutoff_mask);
295 velecsum = _mm_add_pd(velecsum,velec);
299 fscal = _mm_and_pd(fscal,cutoff_mask);
301 /* Update vectorial force */
302 fix2 = _mm_macc_pd(dx20,fscal,fix2);
303 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
304 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
306 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
307 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
308 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
312 /**************************
313 * CALCULATE INTERACTIONS *
314 **************************/
316 if (gmx_mm_any_lt(rsq30,rcutoff2))
319 r30 = _mm_mul_pd(rsq30,rinv30);
321 /* Compute parameters for interactions between i and j atoms */
322 qq30 = _mm_mul_pd(iq3,jq0);
324 /* EWALD ELECTROSTATICS */
326 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
327 ewrt = _mm_mul_pd(r30,ewtabscale);
328 ewitab = _mm_cvttpd_epi32(ewrt);
330 eweps = _mm_frcz_pd(ewrt);
332 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
334 twoeweps = _mm_add_pd(eweps,eweps);
335 ewitab = _mm_slli_epi32(ewitab,2);
336 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
337 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
338 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
339 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
340 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
341 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
342 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
343 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
344 velec = _mm_mul_pd(qq30,_mm_sub_pd(_mm_sub_pd(rinv30,sh_ewald),velec));
345 felec = _mm_mul_pd(_mm_mul_pd(qq30,rinv30),_mm_sub_pd(rinvsq30,felec));
347 cutoff_mask = _mm_cmplt_pd(rsq30,rcutoff2);
349 /* Update potential sum for this i atom from the interaction with this j atom. */
350 velec = _mm_and_pd(velec,cutoff_mask);
351 velecsum = _mm_add_pd(velecsum,velec);
355 fscal = _mm_and_pd(fscal,cutoff_mask);
357 /* Update vectorial force */
358 fix3 = _mm_macc_pd(dx30,fscal,fix3);
359 fiy3 = _mm_macc_pd(dy30,fscal,fiy3);
360 fiz3 = _mm_macc_pd(dz30,fscal,fiz3);
362 fjx0 = _mm_macc_pd(dx30,fscal,fjx0);
363 fjy0 = _mm_macc_pd(dy30,fscal,fjy0);
364 fjz0 = _mm_macc_pd(dz30,fscal,fjz0);
368 gmx_mm_decrement_1rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0);
370 /* Inner loop uses 150 flops */
377 j_coord_offsetA = DIM*jnrA;
379 /* load j atom coordinates */
380 gmx_mm_load_1rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
383 /* Calculate displacement vector */
384 dx10 = _mm_sub_pd(ix1,jx0);
385 dy10 = _mm_sub_pd(iy1,jy0);
386 dz10 = _mm_sub_pd(iz1,jz0);
387 dx20 = _mm_sub_pd(ix2,jx0);
388 dy20 = _mm_sub_pd(iy2,jy0);
389 dz20 = _mm_sub_pd(iz2,jz0);
390 dx30 = _mm_sub_pd(ix3,jx0);
391 dy30 = _mm_sub_pd(iy3,jy0);
392 dz30 = _mm_sub_pd(iz3,jz0);
394 /* Calculate squared distance and things based on it */
395 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
396 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
397 rsq30 = gmx_mm_calc_rsq_pd(dx30,dy30,dz30);
399 rinv10 = gmx_mm_invsqrt_pd(rsq10);
400 rinv20 = gmx_mm_invsqrt_pd(rsq20);
401 rinv30 = gmx_mm_invsqrt_pd(rsq30);
403 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
404 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
405 rinvsq30 = _mm_mul_pd(rinv30,rinv30);
407 /* Load parameters for j particles */
408 jq0 = _mm_load_sd(charge+jnrA+0);
410 fjx0 = _mm_setzero_pd();
411 fjy0 = _mm_setzero_pd();
412 fjz0 = _mm_setzero_pd();
414 /**************************
415 * CALCULATE INTERACTIONS *
416 **************************/
418 if (gmx_mm_any_lt(rsq10,rcutoff2))
421 r10 = _mm_mul_pd(rsq10,rinv10);
423 /* Compute parameters for interactions between i and j atoms */
424 qq10 = _mm_mul_pd(iq1,jq0);
426 /* EWALD ELECTROSTATICS */
428 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
429 ewrt = _mm_mul_pd(r10,ewtabscale);
430 ewitab = _mm_cvttpd_epi32(ewrt);
432 eweps = _mm_frcz_pd(ewrt);
434 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
436 twoeweps = _mm_add_pd(eweps,eweps);
437 ewitab = _mm_slli_epi32(ewitab,2);
438 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
439 ewtabD = _mm_setzero_pd();
440 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
441 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
442 ewtabFn = _mm_setzero_pd();
443 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
444 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
445 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
446 velec = _mm_mul_pd(qq10,_mm_sub_pd(_mm_sub_pd(rinv10,sh_ewald),velec));
447 felec = _mm_mul_pd(_mm_mul_pd(qq10,rinv10),_mm_sub_pd(rinvsq10,felec));
449 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
451 /* Update potential sum for this i atom from the interaction with this j atom. */
452 velec = _mm_and_pd(velec,cutoff_mask);
453 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
454 velecsum = _mm_add_pd(velecsum,velec);
458 fscal = _mm_and_pd(fscal,cutoff_mask);
460 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
462 /* Update vectorial force */
463 fix1 = _mm_macc_pd(dx10,fscal,fix1);
464 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
465 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
467 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
468 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
469 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
473 /**************************
474 * CALCULATE INTERACTIONS *
475 **************************/
477 if (gmx_mm_any_lt(rsq20,rcutoff2))
480 r20 = _mm_mul_pd(rsq20,rinv20);
482 /* Compute parameters for interactions between i and j atoms */
483 qq20 = _mm_mul_pd(iq2,jq0);
485 /* EWALD ELECTROSTATICS */
487 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
488 ewrt = _mm_mul_pd(r20,ewtabscale);
489 ewitab = _mm_cvttpd_epi32(ewrt);
491 eweps = _mm_frcz_pd(ewrt);
493 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
495 twoeweps = _mm_add_pd(eweps,eweps);
496 ewitab = _mm_slli_epi32(ewitab,2);
497 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
498 ewtabD = _mm_setzero_pd();
499 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
500 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
501 ewtabFn = _mm_setzero_pd();
502 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
503 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
504 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
505 velec = _mm_mul_pd(qq20,_mm_sub_pd(_mm_sub_pd(rinv20,sh_ewald),velec));
506 felec = _mm_mul_pd(_mm_mul_pd(qq20,rinv20),_mm_sub_pd(rinvsq20,felec));
508 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
510 /* Update potential sum for this i atom from the interaction with this j atom. */
511 velec = _mm_and_pd(velec,cutoff_mask);
512 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
513 velecsum = _mm_add_pd(velecsum,velec);
517 fscal = _mm_and_pd(fscal,cutoff_mask);
519 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
521 /* Update vectorial force */
522 fix2 = _mm_macc_pd(dx20,fscal,fix2);
523 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
524 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
526 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
527 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
528 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
532 /**************************
533 * CALCULATE INTERACTIONS *
534 **************************/
536 if (gmx_mm_any_lt(rsq30,rcutoff2))
539 r30 = _mm_mul_pd(rsq30,rinv30);
541 /* Compute parameters for interactions between i and j atoms */
542 qq30 = _mm_mul_pd(iq3,jq0);
544 /* EWALD ELECTROSTATICS */
546 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
547 ewrt = _mm_mul_pd(r30,ewtabscale);
548 ewitab = _mm_cvttpd_epi32(ewrt);
550 eweps = _mm_frcz_pd(ewrt);
552 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
554 twoeweps = _mm_add_pd(eweps,eweps);
555 ewitab = _mm_slli_epi32(ewitab,2);
556 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
557 ewtabD = _mm_setzero_pd();
558 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
559 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
560 ewtabFn = _mm_setzero_pd();
561 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
562 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
563 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
564 velec = _mm_mul_pd(qq30,_mm_sub_pd(_mm_sub_pd(rinv30,sh_ewald),velec));
565 felec = _mm_mul_pd(_mm_mul_pd(qq30,rinv30),_mm_sub_pd(rinvsq30,felec));
567 cutoff_mask = _mm_cmplt_pd(rsq30,rcutoff2);
569 /* Update potential sum for this i atom from the interaction with this j atom. */
570 velec = _mm_and_pd(velec,cutoff_mask);
571 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
572 velecsum = _mm_add_pd(velecsum,velec);
576 fscal = _mm_and_pd(fscal,cutoff_mask);
578 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
580 /* Update vectorial force */
581 fix3 = _mm_macc_pd(dx30,fscal,fix3);
582 fiy3 = _mm_macc_pd(dy30,fscal,fiy3);
583 fiz3 = _mm_macc_pd(dz30,fscal,fiz3);
585 fjx0 = _mm_macc_pd(dx30,fscal,fjx0);
586 fjy0 = _mm_macc_pd(dy30,fscal,fjy0);
587 fjz0 = _mm_macc_pd(dz30,fscal,fjz0);
591 gmx_mm_decrement_1rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0);
593 /* Inner loop uses 150 flops */
596 /* End of innermost loop */
598 gmx_mm_update_iforce_3atom_swizzle_pd(fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
599 f+i_coord_offset+DIM,fshift+i_shift_offset);
602 /* Update potential energies */
603 gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
605 /* Increment number of inner iterations */
606 inneriter += j_index_end - j_index_start;
608 /* Outer loop uses 19 flops */
611 /* Increment number of outer iterations */
614 /* Update outer/inner flops */
616 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_W4_VF,outeriter*19 + inneriter*150);
619 * Gromacs nonbonded kernel: nb_kernel_ElecEwSh_VdwNone_GeomW4P1_F_avx_128_fma_double
620 * Electrostatics interaction: Ewald
621 * VdW interaction: None
622 * Geometry: Water4-Particle
623 * Calculate force/pot: Force
626 nb_kernel_ElecEwSh_VdwNone_GeomW4P1_F_avx_128_fma_double
627 (t_nblist * gmx_restrict nlist,
628 rvec * gmx_restrict xx,
629 rvec * gmx_restrict ff,
630 t_forcerec * gmx_restrict fr,
631 t_mdatoms * gmx_restrict mdatoms,
632 nb_kernel_data_t * gmx_restrict kernel_data,
633 t_nrnb * gmx_restrict nrnb)
635 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
636 * just 0 for non-waters.
637 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
638 * jnr indices corresponding to data put in the four positions in the SIMD register.
640 int i_shift_offset,i_coord_offset,outeriter,inneriter;
641 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
643 int j_coord_offsetA,j_coord_offsetB;
644 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
646 real *shiftvec,*fshift,*x,*f;
647 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
649 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
651 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
653 __m128d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
654 int vdwjidx0A,vdwjidx0B;
655 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
656 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
657 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
658 __m128d dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
659 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
662 __m128d ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
664 __m128d dummy_mask,cutoff_mask;
665 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
666 __m128d one = _mm_set1_pd(1.0);
667 __m128d two = _mm_set1_pd(2.0);
673 jindex = nlist->jindex;
675 shiftidx = nlist->shift;
677 shiftvec = fr->shift_vec[0];
678 fshift = fr->fshift[0];
679 facel = _mm_set1_pd(fr->epsfac);
680 charge = mdatoms->chargeA;
682 sh_ewald = _mm_set1_pd(fr->ic->sh_ewald);
683 ewtab = fr->ic->tabq_coul_F;
684 ewtabscale = _mm_set1_pd(fr->ic->tabq_scale);
685 ewtabhalfspace = _mm_set1_pd(0.5/fr->ic->tabq_scale);
687 /* Setup water-specific parameters */
688 inr = nlist->iinr[0];
689 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
690 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
691 iq3 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
693 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
694 rcutoff_scalar = fr->rcoulomb;
695 rcutoff = _mm_set1_pd(rcutoff_scalar);
696 rcutoff2 = _mm_mul_pd(rcutoff,rcutoff);
698 /* Avoid stupid compiler warnings */
706 /* Start outer loop over neighborlists */
707 for(iidx=0; iidx<nri; iidx++)
709 /* Load shift vector for this list */
710 i_shift_offset = DIM*shiftidx[iidx];
712 /* Load limits for loop over neighbors */
713 j_index_start = jindex[iidx];
714 j_index_end = jindex[iidx+1];
716 /* Get outer coordinate index */
718 i_coord_offset = DIM*inr;
720 /* Load i particle coords and add shift vector */
721 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset+DIM,
722 &ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
724 fix1 = _mm_setzero_pd();
725 fiy1 = _mm_setzero_pd();
726 fiz1 = _mm_setzero_pd();
727 fix2 = _mm_setzero_pd();
728 fiy2 = _mm_setzero_pd();
729 fiz2 = _mm_setzero_pd();
730 fix3 = _mm_setzero_pd();
731 fiy3 = _mm_setzero_pd();
732 fiz3 = _mm_setzero_pd();
734 /* Start inner kernel loop */
735 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
738 /* Get j neighbor index, and coordinate index */
741 j_coord_offsetA = DIM*jnrA;
742 j_coord_offsetB = DIM*jnrB;
744 /* load j atom coordinates */
745 gmx_mm_load_1rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
748 /* Calculate displacement vector */
749 dx10 = _mm_sub_pd(ix1,jx0);
750 dy10 = _mm_sub_pd(iy1,jy0);
751 dz10 = _mm_sub_pd(iz1,jz0);
752 dx20 = _mm_sub_pd(ix2,jx0);
753 dy20 = _mm_sub_pd(iy2,jy0);
754 dz20 = _mm_sub_pd(iz2,jz0);
755 dx30 = _mm_sub_pd(ix3,jx0);
756 dy30 = _mm_sub_pd(iy3,jy0);
757 dz30 = _mm_sub_pd(iz3,jz0);
759 /* Calculate squared distance and things based on it */
760 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
761 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
762 rsq30 = gmx_mm_calc_rsq_pd(dx30,dy30,dz30);
764 rinv10 = gmx_mm_invsqrt_pd(rsq10);
765 rinv20 = gmx_mm_invsqrt_pd(rsq20);
766 rinv30 = gmx_mm_invsqrt_pd(rsq30);
768 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
769 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
770 rinvsq30 = _mm_mul_pd(rinv30,rinv30);
772 /* Load parameters for j particles */
773 jq0 = gmx_mm_load_2real_swizzle_pd(charge+jnrA+0,charge+jnrB+0);
775 fjx0 = _mm_setzero_pd();
776 fjy0 = _mm_setzero_pd();
777 fjz0 = _mm_setzero_pd();
779 /**************************
780 * CALCULATE INTERACTIONS *
781 **************************/
783 if (gmx_mm_any_lt(rsq10,rcutoff2))
786 r10 = _mm_mul_pd(rsq10,rinv10);
788 /* Compute parameters for interactions between i and j atoms */
789 qq10 = _mm_mul_pd(iq1,jq0);
791 /* EWALD ELECTROSTATICS */
793 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
794 ewrt = _mm_mul_pd(r10,ewtabscale);
795 ewitab = _mm_cvttpd_epi32(ewrt);
797 eweps = _mm_frcz_pd(ewrt);
799 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
801 twoeweps = _mm_add_pd(eweps,eweps);
802 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
804 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
805 felec = _mm_mul_pd(_mm_mul_pd(qq10,rinv10),_mm_sub_pd(rinvsq10,felec));
807 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
811 fscal = _mm_and_pd(fscal,cutoff_mask);
813 /* Update vectorial force */
814 fix1 = _mm_macc_pd(dx10,fscal,fix1);
815 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
816 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
818 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
819 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
820 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
824 /**************************
825 * CALCULATE INTERACTIONS *
826 **************************/
828 if (gmx_mm_any_lt(rsq20,rcutoff2))
831 r20 = _mm_mul_pd(rsq20,rinv20);
833 /* Compute parameters for interactions between i and j atoms */
834 qq20 = _mm_mul_pd(iq2,jq0);
836 /* EWALD ELECTROSTATICS */
838 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
839 ewrt = _mm_mul_pd(r20,ewtabscale);
840 ewitab = _mm_cvttpd_epi32(ewrt);
842 eweps = _mm_frcz_pd(ewrt);
844 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
846 twoeweps = _mm_add_pd(eweps,eweps);
847 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
849 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
850 felec = _mm_mul_pd(_mm_mul_pd(qq20,rinv20),_mm_sub_pd(rinvsq20,felec));
852 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
856 fscal = _mm_and_pd(fscal,cutoff_mask);
858 /* Update vectorial force */
859 fix2 = _mm_macc_pd(dx20,fscal,fix2);
860 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
861 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
863 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
864 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
865 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
869 /**************************
870 * CALCULATE INTERACTIONS *
871 **************************/
873 if (gmx_mm_any_lt(rsq30,rcutoff2))
876 r30 = _mm_mul_pd(rsq30,rinv30);
878 /* Compute parameters for interactions between i and j atoms */
879 qq30 = _mm_mul_pd(iq3,jq0);
881 /* EWALD ELECTROSTATICS */
883 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
884 ewrt = _mm_mul_pd(r30,ewtabscale);
885 ewitab = _mm_cvttpd_epi32(ewrt);
887 eweps = _mm_frcz_pd(ewrt);
889 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
891 twoeweps = _mm_add_pd(eweps,eweps);
892 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
894 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
895 felec = _mm_mul_pd(_mm_mul_pd(qq30,rinv30),_mm_sub_pd(rinvsq30,felec));
897 cutoff_mask = _mm_cmplt_pd(rsq30,rcutoff2);
901 fscal = _mm_and_pd(fscal,cutoff_mask);
903 /* Update vectorial force */
904 fix3 = _mm_macc_pd(dx30,fscal,fix3);
905 fiy3 = _mm_macc_pd(dy30,fscal,fiy3);
906 fiz3 = _mm_macc_pd(dz30,fscal,fiz3);
908 fjx0 = _mm_macc_pd(dx30,fscal,fjx0);
909 fjy0 = _mm_macc_pd(dy30,fscal,fjy0);
910 fjz0 = _mm_macc_pd(dz30,fscal,fjz0);
914 gmx_mm_decrement_1rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0);
916 /* Inner loop uses 129 flops */
923 j_coord_offsetA = DIM*jnrA;
925 /* load j atom coordinates */
926 gmx_mm_load_1rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
929 /* Calculate displacement vector */
930 dx10 = _mm_sub_pd(ix1,jx0);
931 dy10 = _mm_sub_pd(iy1,jy0);
932 dz10 = _mm_sub_pd(iz1,jz0);
933 dx20 = _mm_sub_pd(ix2,jx0);
934 dy20 = _mm_sub_pd(iy2,jy0);
935 dz20 = _mm_sub_pd(iz2,jz0);
936 dx30 = _mm_sub_pd(ix3,jx0);
937 dy30 = _mm_sub_pd(iy3,jy0);
938 dz30 = _mm_sub_pd(iz3,jz0);
940 /* Calculate squared distance and things based on it */
941 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
942 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
943 rsq30 = gmx_mm_calc_rsq_pd(dx30,dy30,dz30);
945 rinv10 = gmx_mm_invsqrt_pd(rsq10);
946 rinv20 = gmx_mm_invsqrt_pd(rsq20);
947 rinv30 = gmx_mm_invsqrt_pd(rsq30);
949 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
950 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
951 rinvsq30 = _mm_mul_pd(rinv30,rinv30);
953 /* Load parameters for j particles */
954 jq0 = _mm_load_sd(charge+jnrA+0);
956 fjx0 = _mm_setzero_pd();
957 fjy0 = _mm_setzero_pd();
958 fjz0 = _mm_setzero_pd();
960 /**************************
961 * CALCULATE INTERACTIONS *
962 **************************/
964 if (gmx_mm_any_lt(rsq10,rcutoff2))
967 r10 = _mm_mul_pd(rsq10,rinv10);
969 /* Compute parameters for interactions between i and j atoms */
970 qq10 = _mm_mul_pd(iq1,jq0);
972 /* EWALD ELECTROSTATICS */
974 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
975 ewrt = _mm_mul_pd(r10,ewtabscale);
976 ewitab = _mm_cvttpd_epi32(ewrt);
978 eweps = _mm_frcz_pd(ewrt);
980 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
982 twoeweps = _mm_add_pd(eweps,eweps);
983 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
984 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
985 felec = _mm_mul_pd(_mm_mul_pd(qq10,rinv10),_mm_sub_pd(rinvsq10,felec));
987 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
991 fscal = _mm_and_pd(fscal,cutoff_mask);
993 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
995 /* Update vectorial force */
996 fix1 = _mm_macc_pd(dx10,fscal,fix1);
997 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
998 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
1000 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
1001 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
1002 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
1006 /**************************
1007 * CALCULATE INTERACTIONS *
1008 **************************/
1010 if (gmx_mm_any_lt(rsq20,rcutoff2))
1013 r20 = _mm_mul_pd(rsq20,rinv20);
1015 /* Compute parameters for interactions between i and j atoms */
1016 qq20 = _mm_mul_pd(iq2,jq0);
1018 /* EWALD ELECTROSTATICS */
1020 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1021 ewrt = _mm_mul_pd(r20,ewtabscale);
1022 ewitab = _mm_cvttpd_epi32(ewrt);
1024 eweps = _mm_frcz_pd(ewrt);
1026 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1028 twoeweps = _mm_add_pd(eweps,eweps);
1029 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
1030 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1031 felec = _mm_mul_pd(_mm_mul_pd(qq20,rinv20),_mm_sub_pd(rinvsq20,felec));
1033 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
1037 fscal = _mm_and_pd(fscal,cutoff_mask);
1039 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1041 /* Update vectorial force */
1042 fix2 = _mm_macc_pd(dx20,fscal,fix2);
1043 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
1044 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
1046 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
1047 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
1048 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
1052 /**************************
1053 * CALCULATE INTERACTIONS *
1054 **************************/
1056 if (gmx_mm_any_lt(rsq30,rcutoff2))
1059 r30 = _mm_mul_pd(rsq30,rinv30);
1061 /* Compute parameters for interactions between i and j atoms */
1062 qq30 = _mm_mul_pd(iq3,jq0);
1064 /* EWALD ELECTROSTATICS */
1066 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1067 ewrt = _mm_mul_pd(r30,ewtabscale);
1068 ewitab = _mm_cvttpd_epi32(ewrt);
1070 eweps = _mm_frcz_pd(ewrt);
1072 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1074 twoeweps = _mm_add_pd(eweps,eweps);
1075 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
1076 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1077 felec = _mm_mul_pd(_mm_mul_pd(qq30,rinv30),_mm_sub_pd(rinvsq30,felec));
1079 cutoff_mask = _mm_cmplt_pd(rsq30,rcutoff2);
1083 fscal = _mm_and_pd(fscal,cutoff_mask);
1085 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1087 /* Update vectorial force */
1088 fix3 = _mm_macc_pd(dx30,fscal,fix3);
1089 fiy3 = _mm_macc_pd(dy30,fscal,fiy3);
1090 fiz3 = _mm_macc_pd(dz30,fscal,fiz3);
1092 fjx0 = _mm_macc_pd(dx30,fscal,fjx0);
1093 fjy0 = _mm_macc_pd(dy30,fscal,fjy0);
1094 fjz0 = _mm_macc_pd(dz30,fscal,fjz0);
1098 gmx_mm_decrement_1rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0);
1100 /* Inner loop uses 129 flops */
1103 /* End of innermost loop */
1105 gmx_mm_update_iforce_3atom_swizzle_pd(fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1106 f+i_coord_offset+DIM,fshift+i_shift_offset);
1108 /* Increment number of inner iterations */
1109 inneriter += j_index_end - j_index_start;
1111 /* Outer loop uses 18 flops */
1114 /* Increment number of outer iterations */
1117 /* Update outer/inner flops */
1119 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_W4_F,outeriter*18 + inneriter*129);