2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014,2015,2017, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_double kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/gmxlib/nrnb.h"
47 #include "kernelutil_x86_avx_128_fma_double.h"
50 * Gromacs nonbonded kernel: nb_kernel_ElecEw_VdwCSTab_GeomW4W4_VF_avx_128_fma_double
51 * Electrostatics interaction: Ewald
52 * VdW interaction: CubicSplineTable
53 * Geometry: Water4-Water4
54 * Calculate force/pot: PotentialAndForce
57 nb_kernel_ElecEw_VdwCSTab_GeomW4W4_VF_avx_128_fma_double
58 (t_nblist * gmx_restrict nlist,
59 rvec * gmx_restrict xx,
60 rvec * gmx_restrict ff,
61 struct t_forcerec * gmx_restrict fr,
62 t_mdatoms * gmx_restrict mdatoms,
63 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
64 t_nrnb * gmx_restrict nrnb)
66 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
67 * just 0 for non-waters.
68 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
69 * jnr indices corresponding to data put in the four positions in the SIMD register.
71 int i_shift_offset,i_coord_offset,outeriter,inneriter;
72 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
74 int j_coord_offsetA,j_coord_offsetB;
75 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
77 real *shiftvec,*fshift,*x,*f;
78 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
80 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
82 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
84 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
86 __m128d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
87 int vdwjidx0A,vdwjidx0B;
88 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
89 int vdwjidx1A,vdwjidx1B;
90 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
91 int vdwjidx2A,vdwjidx2B;
92 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
93 int vdwjidx3A,vdwjidx3B;
94 __m128d jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
95 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
96 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
97 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
98 __m128d dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
99 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
100 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
101 __m128d dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
102 __m128d dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
103 __m128d dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
104 __m128d dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
105 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
108 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
111 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
112 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
114 __m128i ifour = _mm_set1_epi32(4);
115 __m128d rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF,twovfeps;
118 __m128d ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
120 __m128d dummy_mask,cutoff_mask;
121 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
122 __m128d one = _mm_set1_pd(1.0);
123 __m128d two = _mm_set1_pd(2.0);
129 jindex = nlist->jindex;
131 shiftidx = nlist->shift;
133 shiftvec = fr->shift_vec[0];
134 fshift = fr->fshift[0];
135 facel = _mm_set1_pd(fr->ic->epsfac);
136 charge = mdatoms->chargeA;
137 nvdwtype = fr->ntype;
139 vdwtype = mdatoms->typeA;
141 vftab = kernel_data->table_vdw->data;
142 vftabscale = _mm_set1_pd(kernel_data->table_vdw->scale);
144 sh_ewald = _mm_set1_pd(fr->ic->sh_ewald);
145 ewtab = fr->ic->tabq_coul_FDV0;
146 ewtabscale = _mm_set1_pd(fr->ic->tabq_scale);
147 ewtabhalfspace = _mm_set1_pd(0.5/fr->ic->tabq_scale);
149 /* Setup water-specific parameters */
150 inr = nlist->iinr[0];
151 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
152 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
153 iq3 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
154 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
156 jq1 = _mm_set1_pd(charge[inr+1]);
157 jq2 = _mm_set1_pd(charge[inr+2]);
158 jq3 = _mm_set1_pd(charge[inr+3]);
159 vdwjidx0A = 2*vdwtype[inr+0];
160 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
161 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
162 qq11 = _mm_mul_pd(iq1,jq1);
163 qq12 = _mm_mul_pd(iq1,jq2);
164 qq13 = _mm_mul_pd(iq1,jq3);
165 qq21 = _mm_mul_pd(iq2,jq1);
166 qq22 = _mm_mul_pd(iq2,jq2);
167 qq23 = _mm_mul_pd(iq2,jq3);
168 qq31 = _mm_mul_pd(iq3,jq1);
169 qq32 = _mm_mul_pd(iq3,jq2);
170 qq33 = _mm_mul_pd(iq3,jq3);
172 /* Avoid stupid compiler warnings */
180 /* Start outer loop over neighborlists */
181 for(iidx=0; iidx<nri; iidx++)
183 /* Load shift vector for this list */
184 i_shift_offset = DIM*shiftidx[iidx];
186 /* Load limits for loop over neighbors */
187 j_index_start = jindex[iidx];
188 j_index_end = jindex[iidx+1];
190 /* Get outer coordinate index */
192 i_coord_offset = DIM*inr;
194 /* Load i particle coords and add shift vector */
195 gmx_mm_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
196 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
198 fix0 = _mm_setzero_pd();
199 fiy0 = _mm_setzero_pd();
200 fiz0 = _mm_setzero_pd();
201 fix1 = _mm_setzero_pd();
202 fiy1 = _mm_setzero_pd();
203 fiz1 = _mm_setzero_pd();
204 fix2 = _mm_setzero_pd();
205 fiy2 = _mm_setzero_pd();
206 fiz2 = _mm_setzero_pd();
207 fix3 = _mm_setzero_pd();
208 fiy3 = _mm_setzero_pd();
209 fiz3 = _mm_setzero_pd();
211 /* Reset potential sums */
212 velecsum = _mm_setzero_pd();
213 vvdwsum = _mm_setzero_pd();
215 /* Start inner kernel loop */
216 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
219 /* Get j neighbor index, and coordinate index */
222 j_coord_offsetA = DIM*jnrA;
223 j_coord_offsetB = DIM*jnrB;
225 /* load j atom coordinates */
226 gmx_mm_load_4rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
227 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
228 &jy2,&jz2,&jx3,&jy3,&jz3);
230 /* Calculate displacement vector */
231 dx00 = _mm_sub_pd(ix0,jx0);
232 dy00 = _mm_sub_pd(iy0,jy0);
233 dz00 = _mm_sub_pd(iz0,jz0);
234 dx11 = _mm_sub_pd(ix1,jx1);
235 dy11 = _mm_sub_pd(iy1,jy1);
236 dz11 = _mm_sub_pd(iz1,jz1);
237 dx12 = _mm_sub_pd(ix1,jx2);
238 dy12 = _mm_sub_pd(iy1,jy2);
239 dz12 = _mm_sub_pd(iz1,jz2);
240 dx13 = _mm_sub_pd(ix1,jx3);
241 dy13 = _mm_sub_pd(iy1,jy3);
242 dz13 = _mm_sub_pd(iz1,jz3);
243 dx21 = _mm_sub_pd(ix2,jx1);
244 dy21 = _mm_sub_pd(iy2,jy1);
245 dz21 = _mm_sub_pd(iz2,jz1);
246 dx22 = _mm_sub_pd(ix2,jx2);
247 dy22 = _mm_sub_pd(iy2,jy2);
248 dz22 = _mm_sub_pd(iz2,jz2);
249 dx23 = _mm_sub_pd(ix2,jx3);
250 dy23 = _mm_sub_pd(iy2,jy3);
251 dz23 = _mm_sub_pd(iz2,jz3);
252 dx31 = _mm_sub_pd(ix3,jx1);
253 dy31 = _mm_sub_pd(iy3,jy1);
254 dz31 = _mm_sub_pd(iz3,jz1);
255 dx32 = _mm_sub_pd(ix3,jx2);
256 dy32 = _mm_sub_pd(iy3,jy2);
257 dz32 = _mm_sub_pd(iz3,jz2);
258 dx33 = _mm_sub_pd(ix3,jx3);
259 dy33 = _mm_sub_pd(iy3,jy3);
260 dz33 = _mm_sub_pd(iz3,jz3);
262 /* Calculate squared distance and things based on it */
263 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
264 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
265 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
266 rsq13 = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
267 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
268 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
269 rsq23 = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
270 rsq31 = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
271 rsq32 = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
272 rsq33 = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
274 rinv00 = avx128fma_invsqrt_d(rsq00);
275 rinv11 = avx128fma_invsqrt_d(rsq11);
276 rinv12 = avx128fma_invsqrt_d(rsq12);
277 rinv13 = avx128fma_invsqrt_d(rsq13);
278 rinv21 = avx128fma_invsqrt_d(rsq21);
279 rinv22 = avx128fma_invsqrt_d(rsq22);
280 rinv23 = avx128fma_invsqrt_d(rsq23);
281 rinv31 = avx128fma_invsqrt_d(rsq31);
282 rinv32 = avx128fma_invsqrt_d(rsq32);
283 rinv33 = avx128fma_invsqrt_d(rsq33);
285 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
286 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
287 rinvsq13 = _mm_mul_pd(rinv13,rinv13);
288 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
289 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
290 rinvsq23 = _mm_mul_pd(rinv23,rinv23);
291 rinvsq31 = _mm_mul_pd(rinv31,rinv31);
292 rinvsq32 = _mm_mul_pd(rinv32,rinv32);
293 rinvsq33 = _mm_mul_pd(rinv33,rinv33);
295 fjx0 = _mm_setzero_pd();
296 fjy0 = _mm_setzero_pd();
297 fjz0 = _mm_setzero_pd();
298 fjx1 = _mm_setzero_pd();
299 fjy1 = _mm_setzero_pd();
300 fjz1 = _mm_setzero_pd();
301 fjx2 = _mm_setzero_pd();
302 fjy2 = _mm_setzero_pd();
303 fjz2 = _mm_setzero_pd();
304 fjx3 = _mm_setzero_pd();
305 fjy3 = _mm_setzero_pd();
306 fjz3 = _mm_setzero_pd();
308 /**************************
309 * CALCULATE INTERACTIONS *
310 **************************/
312 r00 = _mm_mul_pd(rsq00,rinv00);
314 /* Calculate table index by multiplying r with table scale and truncate to integer */
315 rt = _mm_mul_pd(r00,vftabscale);
316 vfitab = _mm_cvttpd_epi32(rt);
318 vfeps = _mm_frcz_pd(rt);
320 vfeps = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
322 twovfeps = _mm_add_pd(vfeps,vfeps);
323 vfitab = _mm_slli_epi32(vfitab,3);
325 /* CUBIC SPLINE TABLE DISPERSION */
326 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
327 F = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
328 GMX_MM_TRANSPOSE2_PD(Y,F);
329 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
330 H = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
331 GMX_MM_TRANSPOSE2_PD(G,H);
332 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
333 VV = _mm_macc_pd(vfeps,Fp,Y);
334 vvdw6 = _mm_mul_pd(c6_00,VV);
335 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
336 fvdw6 = _mm_mul_pd(c6_00,FF);
338 /* CUBIC SPLINE TABLE REPULSION */
339 vfitab = _mm_add_epi32(vfitab,ifour);
340 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
341 F = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
342 GMX_MM_TRANSPOSE2_PD(Y,F);
343 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
344 H = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
345 GMX_MM_TRANSPOSE2_PD(G,H);
346 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
347 VV = _mm_macc_pd(vfeps,Fp,Y);
348 vvdw12 = _mm_mul_pd(c12_00,VV);
349 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
350 fvdw12 = _mm_mul_pd(c12_00,FF);
351 vvdw = _mm_add_pd(vvdw12,vvdw6);
352 fvdw = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
354 /* Update potential sum for this i atom from the interaction with this j atom. */
355 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
359 /* Update vectorial force */
360 fix0 = _mm_macc_pd(dx00,fscal,fix0);
361 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
362 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
364 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
365 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
366 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
368 /**************************
369 * CALCULATE INTERACTIONS *
370 **************************/
372 r11 = _mm_mul_pd(rsq11,rinv11);
374 /* EWALD ELECTROSTATICS */
376 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
377 ewrt = _mm_mul_pd(r11,ewtabscale);
378 ewitab = _mm_cvttpd_epi32(ewrt);
380 eweps = _mm_frcz_pd(ewrt);
382 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
384 twoeweps = _mm_add_pd(eweps,eweps);
385 ewitab = _mm_slli_epi32(ewitab,2);
386 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
387 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
388 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
389 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
390 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
391 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
392 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
393 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
394 velec = _mm_mul_pd(qq11,_mm_sub_pd(rinv11,velec));
395 felec = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
397 /* Update potential sum for this i atom from the interaction with this j atom. */
398 velecsum = _mm_add_pd(velecsum,velec);
402 /* Update vectorial force */
403 fix1 = _mm_macc_pd(dx11,fscal,fix1);
404 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
405 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
407 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
408 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
409 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
411 /**************************
412 * CALCULATE INTERACTIONS *
413 **************************/
415 r12 = _mm_mul_pd(rsq12,rinv12);
417 /* EWALD ELECTROSTATICS */
419 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
420 ewrt = _mm_mul_pd(r12,ewtabscale);
421 ewitab = _mm_cvttpd_epi32(ewrt);
423 eweps = _mm_frcz_pd(ewrt);
425 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
427 twoeweps = _mm_add_pd(eweps,eweps);
428 ewitab = _mm_slli_epi32(ewitab,2);
429 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
430 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
431 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
432 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
433 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
434 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
435 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
436 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
437 velec = _mm_mul_pd(qq12,_mm_sub_pd(rinv12,velec));
438 felec = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
440 /* Update potential sum for this i atom from the interaction with this j atom. */
441 velecsum = _mm_add_pd(velecsum,velec);
445 /* Update vectorial force */
446 fix1 = _mm_macc_pd(dx12,fscal,fix1);
447 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
448 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
450 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
451 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
452 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
454 /**************************
455 * CALCULATE INTERACTIONS *
456 **************************/
458 r13 = _mm_mul_pd(rsq13,rinv13);
460 /* EWALD ELECTROSTATICS */
462 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
463 ewrt = _mm_mul_pd(r13,ewtabscale);
464 ewitab = _mm_cvttpd_epi32(ewrt);
466 eweps = _mm_frcz_pd(ewrt);
468 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
470 twoeweps = _mm_add_pd(eweps,eweps);
471 ewitab = _mm_slli_epi32(ewitab,2);
472 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
473 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
474 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
475 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
476 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
477 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
478 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
479 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
480 velec = _mm_mul_pd(qq13,_mm_sub_pd(rinv13,velec));
481 felec = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
483 /* Update potential sum for this i atom from the interaction with this j atom. */
484 velecsum = _mm_add_pd(velecsum,velec);
488 /* Update vectorial force */
489 fix1 = _mm_macc_pd(dx13,fscal,fix1);
490 fiy1 = _mm_macc_pd(dy13,fscal,fiy1);
491 fiz1 = _mm_macc_pd(dz13,fscal,fiz1);
493 fjx3 = _mm_macc_pd(dx13,fscal,fjx3);
494 fjy3 = _mm_macc_pd(dy13,fscal,fjy3);
495 fjz3 = _mm_macc_pd(dz13,fscal,fjz3);
497 /**************************
498 * CALCULATE INTERACTIONS *
499 **************************/
501 r21 = _mm_mul_pd(rsq21,rinv21);
503 /* EWALD ELECTROSTATICS */
505 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
506 ewrt = _mm_mul_pd(r21,ewtabscale);
507 ewitab = _mm_cvttpd_epi32(ewrt);
509 eweps = _mm_frcz_pd(ewrt);
511 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
513 twoeweps = _mm_add_pd(eweps,eweps);
514 ewitab = _mm_slli_epi32(ewitab,2);
515 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
516 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
517 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
518 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
519 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
520 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
521 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
522 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
523 velec = _mm_mul_pd(qq21,_mm_sub_pd(rinv21,velec));
524 felec = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
526 /* Update potential sum for this i atom from the interaction with this j atom. */
527 velecsum = _mm_add_pd(velecsum,velec);
531 /* Update vectorial force */
532 fix2 = _mm_macc_pd(dx21,fscal,fix2);
533 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
534 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
536 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
537 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
538 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
540 /**************************
541 * CALCULATE INTERACTIONS *
542 **************************/
544 r22 = _mm_mul_pd(rsq22,rinv22);
546 /* EWALD ELECTROSTATICS */
548 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
549 ewrt = _mm_mul_pd(r22,ewtabscale);
550 ewitab = _mm_cvttpd_epi32(ewrt);
552 eweps = _mm_frcz_pd(ewrt);
554 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
556 twoeweps = _mm_add_pd(eweps,eweps);
557 ewitab = _mm_slli_epi32(ewitab,2);
558 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
559 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
560 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
561 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
562 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
563 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
564 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
565 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
566 velec = _mm_mul_pd(qq22,_mm_sub_pd(rinv22,velec));
567 felec = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
569 /* Update potential sum for this i atom from the interaction with this j atom. */
570 velecsum = _mm_add_pd(velecsum,velec);
574 /* Update vectorial force */
575 fix2 = _mm_macc_pd(dx22,fscal,fix2);
576 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
577 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
579 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
580 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
581 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
583 /**************************
584 * CALCULATE INTERACTIONS *
585 **************************/
587 r23 = _mm_mul_pd(rsq23,rinv23);
589 /* EWALD ELECTROSTATICS */
591 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
592 ewrt = _mm_mul_pd(r23,ewtabscale);
593 ewitab = _mm_cvttpd_epi32(ewrt);
595 eweps = _mm_frcz_pd(ewrt);
597 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
599 twoeweps = _mm_add_pd(eweps,eweps);
600 ewitab = _mm_slli_epi32(ewitab,2);
601 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
602 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
603 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
604 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
605 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
606 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
607 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
608 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
609 velec = _mm_mul_pd(qq23,_mm_sub_pd(rinv23,velec));
610 felec = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
612 /* Update potential sum for this i atom from the interaction with this j atom. */
613 velecsum = _mm_add_pd(velecsum,velec);
617 /* Update vectorial force */
618 fix2 = _mm_macc_pd(dx23,fscal,fix2);
619 fiy2 = _mm_macc_pd(dy23,fscal,fiy2);
620 fiz2 = _mm_macc_pd(dz23,fscal,fiz2);
622 fjx3 = _mm_macc_pd(dx23,fscal,fjx3);
623 fjy3 = _mm_macc_pd(dy23,fscal,fjy3);
624 fjz3 = _mm_macc_pd(dz23,fscal,fjz3);
626 /**************************
627 * CALCULATE INTERACTIONS *
628 **************************/
630 r31 = _mm_mul_pd(rsq31,rinv31);
632 /* EWALD ELECTROSTATICS */
634 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
635 ewrt = _mm_mul_pd(r31,ewtabscale);
636 ewitab = _mm_cvttpd_epi32(ewrt);
638 eweps = _mm_frcz_pd(ewrt);
640 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
642 twoeweps = _mm_add_pd(eweps,eweps);
643 ewitab = _mm_slli_epi32(ewitab,2);
644 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
645 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
646 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
647 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
648 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
649 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
650 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
651 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
652 velec = _mm_mul_pd(qq31,_mm_sub_pd(rinv31,velec));
653 felec = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
655 /* Update potential sum for this i atom from the interaction with this j atom. */
656 velecsum = _mm_add_pd(velecsum,velec);
660 /* Update vectorial force */
661 fix3 = _mm_macc_pd(dx31,fscal,fix3);
662 fiy3 = _mm_macc_pd(dy31,fscal,fiy3);
663 fiz3 = _mm_macc_pd(dz31,fscal,fiz3);
665 fjx1 = _mm_macc_pd(dx31,fscal,fjx1);
666 fjy1 = _mm_macc_pd(dy31,fscal,fjy1);
667 fjz1 = _mm_macc_pd(dz31,fscal,fjz1);
669 /**************************
670 * CALCULATE INTERACTIONS *
671 **************************/
673 r32 = _mm_mul_pd(rsq32,rinv32);
675 /* EWALD ELECTROSTATICS */
677 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
678 ewrt = _mm_mul_pd(r32,ewtabscale);
679 ewitab = _mm_cvttpd_epi32(ewrt);
681 eweps = _mm_frcz_pd(ewrt);
683 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
685 twoeweps = _mm_add_pd(eweps,eweps);
686 ewitab = _mm_slli_epi32(ewitab,2);
687 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
688 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
689 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
690 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
691 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
692 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
693 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
694 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
695 velec = _mm_mul_pd(qq32,_mm_sub_pd(rinv32,velec));
696 felec = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
698 /* Update potential sum for this i atom from the interaction with this j atom. */
699 velecsum = _mm_add_pd(velecsum,velec);
703 /* Update vectorial force */
704 fix3 = _mm_macc_pd(dx32,fscal,fix3);
705 fiy3 = _mm_macc_pd(dy32,fscal,fiy3);
706 fiz3 = _mm_macc_pd(dz32,fscal,fiz3);
708 fjx2 = _mm_macc_pd(dx32,fscal,fjx2);
709 fjy2 = _mm_macc_pd(dy32,fscal,fjy2);
710 fjz2 = _mm_macc_pd(dz32,fscal,fjz2);
712 /**************************
713 * CALCULATE INTERACTIONS *
714 **************************/
716 r33 = _mm_mul_pd(rsq33,rinv33);
718 /* EWALD ELECTROSTATICS */
720 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
721 ewrt = _mm_mul_pd(r33,ewtabscale);
722 ewitab = _mm_cvttpd_epi32(ewrt);
724 eweps = _mm_frcz_pd(ewrt);
726 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
728 twoeweps = _mm_add_pd(eweps,eweps);
729 ewitab = _mm_slli_epi32(ewitab,2);
730 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
731 ewtabD = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,1) );
732 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
733 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
734 ewtabFn = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,1) +2);
735 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
736 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
737 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
738 velec = _mm_mul_pd(qq33,_mm_sub_pd(rinv33,velec));
739 felec = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
741 /* Update potential sum for this i atom from the interaction with this j atom. */
742 velecsum = _mm_add_pd(velecsum,velec);
746 /* Update vectorial force */
747 fix3 = _mm_macc_pd(dx33,fscal,fix3);
748 fiy3 = _mm_macc_pd(dy33,fscal,fiy3);
749 fiz3 = _mm_macc_pd(dz33,fscal,fiz3);
751 fjx3 = _mm_macc_pd(dx33,fscal,fjx3);
752 fjy3 = _mm_macc_pd(dy33,fscal,fjy3);
753 fjz3 = _mm_macc_pd(dz33,fscal,fjz3);
755 gmx_mm_decrement_4rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
757 /* Inner loop uses 458 flops */
764 j_coord_offsetA = DIM*jnrA;
766 /* load j atom coordinates */
767 gmx_mm_load_4rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
768 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
769 &jy2,&jz2,&jx3,&jy3,&jz3);
771 /* Calculate displacement vector */
772 dx00 = _mm_sub_pd(ix0,jx0);
773 dy00 = _mm_sub_pd(iy0,jy0);
774 dz00 = _mm_sub_pd(iz0,jz0);
775 dx11 = _mm_sub_pd(ix1,jx1);
776 dy11 = _mm_sub_pd(iy1,jy1);
777 dz11 = _mm_sub_pd(iz1,jz1);
778 dx12 = _mm_sub_pd(ix1,jx2);
779 dy12 = _mm_sub_pd(iy1,jy2);
780 dz12 = _mm_sub_pd(iz1,jz2);
781 dx13 = _mm_sub_pd(ix1,jx3);
782 dy13 = _mm_sub_pd(iy1,jy3);
783 dz13 = _mm_sub_pd(iz1,jz3);
784 dx21 = _mm_sub_pd(ix2,jx1);
785 dy21 = _mm_sub_pd(iy2,jy1);
786 dz21 = _mm_sub_pd(iz2,jz1);
787 dx22 = _mm_sub_pd(ix2,jx2);
788 dy22 = _mm_sub_pd(iy2,jy2);
789 dz22 = _mm_sub_pd(iz2,jz2);
790 dx23 = _mm_sub_pd(ix2,jx3);
791 dy23 = _mm_sub_pd(iy2,jy3);
792 dz23 = _mm_sub_pd(iz2,jz3);
793 dx31 = _mm_sub_pd(ix3,jx1);
794 dy31 = _mm_sub_pd(iy3,jy1);
795 dz31 = _mm_sub_pd(iz3,jz1);
796 dx32 = _mm_sub_pd(ix3,jx2);
797 dy32 = _mm_sub_pd(iy3,jy2);
798 dz32 = _mm_sub_pd(iz3,jz2);
799 dx33 = _mm_sub_pd(ix3,jx3);
800 dy33 = _mm_sub_pd(iy3,jy3);
801 dz33 = _mm_sub_pd(iz3,jz3);
803 /* Calculate squared distance and things based on it */
804 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
805 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
806 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
807 rsq13 = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
808 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
809 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
810 rsq23 = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
811 rsq31 = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
812 rsq32 = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
813 rsq33 = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
815 rinv00 = avx128fma_invsqrt_d(rsq00);
816 rinv11 = avx128fma_invsqrt_d(rsq11);
817 rinv12 = avx128fma_invsqrt_d(rsq12);
818 rinv13 = avx128fma_invsqrt_d(rsq13);
819 rinv21 = avx128fma_invsqrt_d(rsq21);
820 rinv22 = avx128fma_invsqrt_d(rsq22);
821 rinv23 = avx128fma_invsqrt_d(rsq23);
822 rinv31 = avx128fma_invsqrt_d(rsq31);
823 rinv32 = avx128fma_invsqrt_d(rsq32);
824 rinv33 = avx128fma_invsqrt_d(rsq33);
826 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
827 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
828 rinvsq13 = _mm_mul_pd(rinv13,rinv13);
829 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
830 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
831 rinvsq23 = _mm_mul_pd(rinv23,rinv23);
832 rinvsq31 = _mm_mul_pd(rinv31,rinv31);
833 rinvsq32 = _mm_mul_pd(rinv32,rinv32);
834 rinvsq33 = _mm_mul_pd(rinv33,rinv33);
836 fjx0 = _mm_setzero_pd();
837 fjy0 = _mm_setzero_pd();
838 fjz0 = _mm_setzero_pd();
839 fjx1 = _mm_setzero_pd();
840 fjy1 = _mm_setzero_pd();
841 fjz1 = _mm_setzero_pd();
842 fjx2 = _mm_setzero_pd();
843 fjy2 = _mm_setzero_pd();
844 fjz2 = _mm_setzero_pd();
845 fjx3 = _mm_setzero_pd();
846 fjy3 = _mm_setzero_pd();
847 fjz3 = _mm_setzero_pd();
849 /**************************
850 * CALCULATE INTERACTIONS *
851 **************************/
853 r00 = _mm_mul_pd(rsq00,rinv00);
855 /* Calculate table index by multiplying r with table scale and truncate to integer */
856 rt = _mm_mul_pd(r00,vftabscale);
857 vfitab = _mm_cvttpd_epi32(rt);
859 vfeps = _mm_frcz_pd(rt);
861 vfeps = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
863 twovfeps = _mm_add_pd(vfeps,vfeps);
864 vfitab = _mm_slli_epi32(vfitab,3);
866 /* CUBIC SPLINE TABLE DISPERSION */
867 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
868 F = _mm_setzero_pd();
869 GMX_MM_TRANSPOSE2_PD(Y,F);
870 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
871 H = _mm_setzero_pd();
872 GMX_MM_TRANSPOSE2_PD(G,H);
873 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
874 VV = _mm_macc_pd(vfeps,Fp,Y);
875 vvdw6 = _mm_mul_pd(c6_00,VV);
876 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
877 fvdw6 = _mm_mul_pd(c6_00,FF);
879 /* CUBIC SPLINE TABLE REPULSION */
880 vfitab = _mm_add_epi32(vfitab,ifour);
881 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
882 F = _mm_setzero_pd();
883 GMX_MM_TRANSPOSE2_PD(Y,F);
884 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
885 H = _mm_setzero_pd();
886 GMX_MM_TRANSPOSE2_PD(G,H);
887 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
888 VV = _mm_macc_pd(vfeps,Fp,Y);
889 vvdw12 = _mm_mul_pd(c12_00,VV);
890 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
891 fvdw12 = _mm_mul_pd(c12_00,FF);
892 vvdw = _mm_add_pd(vvdw12,vvdw6);
893 fvdw = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
895 /* Update potential sum for this i atom from the interaction with this j atom. */
896 vvdw = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
897 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
901 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
903 /* Update vectorial force */
904 fix0 = _mm_macc_pd(dx00,fscal,fix0);
905 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
906 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
908 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
909 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
910 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
912 /**************************
913 * CALCULATE INTERACTIONS *
914 **************************/
916 r11 = _mm_mul_pd(rsq11,rinv11);
918 /* EWALD ELECTROSTATICS */
920 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
921 ewrt = _mm_mul_pd(r11,ewtabscale);
922 ewitab = _mm_cvttpd_epi32(ewrt);
924 eweps = _mm_frcz_pd(ewrt);
926 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
928 twoeweps = _mm_add_pd(eweps,eweps);
929 ewitab = _mm_slli_epi32(ewitab,2);
930 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
931 ewtabD = _mm_setzero_pd();
932 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
933 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
934 ewtabFn = _mm_setzero_pd();
935 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
936 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
937 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
938 velec = _mm_mul_pd(qq11,_mm_sub_pd(rinv11,velec));
939 felec = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
941 /* Update potential sum for this i atom from the interaction with this j atom. */
942 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
943 velecsum = _mm_add_pd(velecsum,velec);
947 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
949 /* Update vectorial force */
950 fix1 = _mm_macc_pd(dx11,fscal,fix1);
951 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
952 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
954 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
955 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
956 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
958 /**************************
959 * CALCULATE INTERACTIONS *
960 **************************/
962 r12 = _mm_mul_pd(rsq12,rinv12);
964 /* EWALD ELECTROSTATICS */
966 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
967 ewrt = _mm_mul_pd(r12,ewtabscale);
968 ewitab = _mm_cvttpd_epi32(ewrt);
970 eweps = _mm_frcz_pd(ewrt);
972 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
974 twoeweps = _mm_add_pd(eweps,eweps);
975 ewitab = _mm_slli_epi32(ewitab,2);
976 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
977 ewtabD = _mm_setzero_pd();
978 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
979 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
980 ewtabFn = _mm_setzero_pd();
981 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
982 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
983 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
984 velec = _mm_mul_pd(qq12,_mm_sub_pd(rinv12,velec));
985 felec = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
987 /* Update potential sum for this i atom from the interaction with this j atom. */
988 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
989 velecsum = _mm_add_pd(velecsum,velec);
993 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
995 /* Update vectorial force */
996 fix1 = _mm_macc_pd(dx12,fscal,fix1);
997 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
998 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1000 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1001 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1002 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1004 /**************************
1005 * CALCULATE INTERACTIONS *
1006 **************************/
1008 r13 = _mm_mul_pd(rsq13,rinv13);
1010 /* EWALD ELECTROSTATICS */
1012 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1013 ewrt = _mm_mul_pd(r13,ewtabscale);
1014 ewitab = _mm_cvttpd_epi32(ewrt);
1016 eweps = _mm_frcz_pd(ewrt);
1018 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1020 twoeweps = _mm_add_pd(eweps,eweps);
1021 ewitab = _mm_slli_epi32(ewitab,2);
1022 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1023 ewtabD = _mm_setzero_pd();
1024 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1025 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1026 ewtabFn = _mm_setzero_pd();
1027 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1028 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1029 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1030 velec = _mm_mul_pd(qq13,_mm_sub_pd(rinv13,velec));
1031 felec = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
1033 /* Update potential sum for this i atom from the interaction with this j atom. */
1034 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1035 velecsum = _mm_add_pd(velecsum,velec);
1039 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1041 /* Update vectorial force */
1042 fix1 = _mm_macc_pd(dx13,fscal,fix1);
1043 fiy1 = _mm_macc_pd(dy13,fscal,fiy1);
1044 fiz1 = _mm_macc_pd(dz13,fscal,fiz1);
1046 fjx3 = _mm_macc_pd(dx13,fscal,fjx3);
1047 fjy3 = _mm_macc_pd(dy13,fscal,fjy3);
1048 fjz3 = _mm_macc_pd(dz13,fscal,fjz3);
1050 /**************************
1051 * CALCULATE INTERACTIONS *
1052 **************************/
1054 r21 = _mm_mul_pd(rsq21,rinv21);
1056 /* EWALD ELECTROSTATICS */
1058 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1059 ewrt = _mm_mul_pd(r21,ewtabscale);
1060 ewitab = _mm_cvttpd_epi32(ewrt);
1062 eweps = _mm_frcz_pd(ewrt);
1064 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1066 twoeweps = _mm_add_pd(eweps,eweps);
1067 ewitab = _mm_slli_epi32(ewitab,2);
1068 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1069 ewtabD = _mm_setzero_pd();
1070 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1071 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1072 ewtabFn = _mm_setzero_pd();
1073 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1074 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1075 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1076 velec = _mm_mul_pd(qq21,_mm_sub_pd(rinv21,velec));
1077 felec = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
1079 /* Update potential sum for this i atom from the interaction with this j atom. */
1080 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1081 velecsum = _mm_add_pd(velecsum,velec);
1085 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1087 /* Update vectorial force */
1088 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1089 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1090 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1092 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1093 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1094 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1096 /**************************
1097 * CALCULATE INTERACTIONS *
1098 **************************/
1100 r22 = _mm_mul_pd(rsq22,rinv22);
1102 /* EWALD ELECTROSTATICS */
1104 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1105 ewrt = _mm_mul_pd(r22,ewtabscale);
1106 ewitab = _mm_cvttpd_epi32(ewrt);
1108 eweps = _mm_frcz_pd(ewrt);
1110 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1112 twoeweps = _mm_add_pd(eweps,eweps);
1113 ewitab = _mm_slli_epi32(ewitab,2);
1114 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1115 ewtabD = _mm_setzero_pd();
1116 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1117 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1118 ewtabFn = _mm_setzero_pd();
1119 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1120 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1121 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1122 velec = _mm_mul_pd(qq22,_mm_sub_pd(rinv22,velec));
1123 felec = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
1125 /* Update potential sum for this i atom from the interaction with this j atom. */
1126 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1127 velecsum = _mm_add_pd(velecsum,velec);
1131 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1133 /* Update vectorial force */
1134 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1135 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1136 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1138 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1139 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1140 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1142 /**************************
1143 * CALCULATE INTERACTIONS *
1144 **************************/
1146 r23 = _mm_mul_pd(rsq23,rinv23);
1148 /* EWALD ELECTROSTATICS */
1150 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1151 ewrt = _mm_mul_pd(r23,ewtabscale);
1152 ewitab = _mm_cvttpd_epi32(ewrt);
1154 eweps = _mm_frcz_pd(ewrt);
1156 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1158 twoeweps = _mm_add_pd(eweps,eweps);
1159 ewitab = _mm_slli_epi32(ewitab,2);
1160 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1161 ewtabD = _mm_setzero_pd();
1162 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1163 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1164 ewtabFn = _mm_setzero_pd();
1165 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1166 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1167 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1168 velec = _mm_mul_pd(qq23,_mm_sub_pd(rinv23,velec));
1169 felec = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
1171 /* Update potential sum for this i atom from the interaction with this j atom. */
1172 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1173 velecsum = _mm_add_pd(velecsum,velec);
1177 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1179 /* Update vectorial force */
1180 fix2 = _mm_macc_pd(dx23,fscal,fix2);
1181 fiy2 = _mm_macc_pd(dy23,fscal,fiy2);
1182 fiz2 = _mm_macc_pd(dz23,fscal,fiz2);
1184 fjx3 = _mm_macc_pd(dx23,fscal,fjx3);
1185 fjy3 = _mm_macc_pd(dy23,fscal,fjy3);
1186 fjz3 = _mm_macc_pd(dz23,fscal,fjz3);
1188 /**************************
1189 * CALCULATE INTERACTIONS *
1190 **************************/
1192 r31 = _mm_mul_pd(rsq31,rinv31);
1194 /* EWALD ELECTROSTATICS */
1196 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1197 ewrt = _mm_mul_pd(r31,ewtabscale);
1198 ewitab = _mm_cvttpd_epi32(ewrt);
1200 eweps = _mm_frcz_pd(ewrt);
1202 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1204 twoeweps = _mm_add_pd(eweps,eweps);
1205 ewitab = _mm_slli_epi32(ewitab,2);
1206 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1207 ewtabD = _mm_setzero_pd();
1208 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1209 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1210 ewtabFn = _mm_setzero_pd();
1211 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1212 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1213 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1214 velec = _mm_mul_pd(qq31,_mm_sub_pd(rinv31,velec));
1215 felec = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
1217 /* Update potential sum for this i atom from the interaction with this j atom. */
1218 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1219 velecsum = _mm_add_pd(velecsum,velec);
1223 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1225 /* Update vectorial force */
1226 fix3 = _mm_macc_pd(dx31,fscal,fix3);
1227 fiy3 = _mm_macc_pd(dy31,fscal,fiy3);
1228 fiz3 = _mm_macc_pd(dz31,fscal,fiz3);
1230 fjx1 = _mm_macc_pd(dx31,fscal,fjx1);
1231 fjy1 = _mm_macc_pd(dy31,fscal,fjy1);
1232 fjz1 = _mm_macc_pd(dz31,fscal,fjz1);
1234 /**************************
1235 * CALCULATE INTERACTIONS *
1236 **************************/
1238 r32 = _mm_mul_pd(rsq32,rinv32);
1240 /* EWALD ELECTROSTATICS */
1242 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1243 ewrt = _mm_mul_pd(r32,ewtabscale);
1244 ewitab = _mm_cvttpd_epi32(ewrt);
1246 eweps = _mm_frcz_pd(ewrt);
1248 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1250 twoeweps = _mm_add_pd(eweps,eweps);
1251 ewitab = _mm_slli_epi32(ewitab,2);
1252 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1253 ewtabD = _mm_setzero_pd();
1254 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1255 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1256 ewtabFn = _mm_setzero_pd();
1257 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1258 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1259 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1260 velec = _mm_mul_pd(qq32,_mm_sub_pd(rinv32,velec));
1261 felec = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
1263 /* Update potential sum for this i atom from the interaction with this j atom. */
1264 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1265 velecsum = _mm_add_pd(velecsum,velec);
1269 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1271 /* Update vectorial force */
1272 fix3 = _mm_macc_pd(dx32,fscal,fix3);
1273 fiy3 = _mm_macc_pd(dy32,fscal,fiy3);
1274 fiz3 = _mm_macc_pd(dz32,fscal,fiz3);
1276 fjx2 = _mm_macc_pd(dx32,fscal,fjx2);
1277 fjy2 = _mm_macc_pd(dy32,fscal,fjy2);
1278 fjz2 = _mm_macc_pd(dz32,fscal,fjz2);
1280 /**************************
1281 * CALCULATE INTERACTIONS *
1282 **************************/
1284 r33 = _mm_mul_pd(rsq33,rinv33);
1286 /* EWALD ELECTROSTATICS */
1288 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1289 ewrt = _mm_mul_pd(r33,ewtabscale);
1290 ewitab = _mm_cvttpd_epi32(ewrt);
1292 eweps = _mm_frcz_pd(ewrt);
1294 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1296 twoeweps = _mm_add_pd(eweps,eweps);
1297 ewitab = _mm_slli_epi32(ewitab,2);
1298 ewtabF = _mm_load_pd( ewtab + _mm_extract_epi32(ewitab,0) );
1299 ewtabD = _mm_setzero_pd();
1300 GMX_MM_TRANSPOSE2_PD(ewtabF,ewtabD);
1301 ewtabV = _mm_load_sd( ewtab + _mm_extract_epi32(ewitab,0) +2);
1302 ewtabFn = _mm_setzero_pd();
1303 GMX_MM_TRANSPOSE2_PD(ewtabV,ewtabFn);
1304 felec = _mm_macc_pd(eweps,ewtabD,ewtabF);
1305 velec = _mm_nmacc_pd(_mm_mul_pd(ewtabhalfspace,eweps) ,_mm_add_pd(ewtabF,felec), ewtabV);
1306 velec = _mm_mul_pd(qq33,_mm_sub_pd(rinv33,velec));
1307 felec = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
1309 /* Update potential sum for this i atom from the interaction with this j atom. */
1310 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1311 velecsum = _mm_add_pd(velecsum,velec);
1315 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1317 /* Update vectorial force */
1318 fix3 = _mm_macc_pd(dx33,fscal,fix3);
1319 fiy3 = _mm_macc_pd(dy33,fscal,fiy3);
1320 fiz3 = _mm_macc_pd(dz33,fscal,fiz3);
1322 fjx3 = _mm_macc_pd(dx33,fscal,fjx3);
1323 fjy3 = _mm_macc_pd(dy33,fscal,fjy3);
1324 fjz3 = _mm_macc_pd(dz33,fscal,fjz3);
1326 gmx_mm_decrement_4rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1328 /* Inner loop uses 458 flops */
1331 /* End of innermost loop */
1333 gmx_mm_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1334 f+i_coord_offset,fshift+i_shift_offset);
1337 /* Update potential energies */
1338 gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
1339 gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
1341 /* Increment number of inner iterations */
1342 inneriter += j_index_end - j_index_start;
1344 /* Outer loop uses 26 flops */
1347 /* Increment number of outer iterations */
1350 /* Update outer/inner flops */
1352 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_VF,outeriter*26 + inneriter*458);
1355 * Gromacs nonbonded kernel: nb_kernel_ElecEw_VdwCSTab_GeomW4W4_F_avx_128_fma_double
1356 * Electrostatics interaction: Ewald
1357 * VdW interaction: CubicSplineTable
1358 * Geometry: Water4-Water4
1359 * Calculate force/pot: Force
1362 nb_kernel_ElecEw_VdwCSTab_GeomW4W4_F_avx_128_fma_double
1363 (t_nblist * gmx_restrict nlist,
1364 rvec * gmx_restrict xx,
1365 rvec * gmx_restrict ff,
1366 struct t_forcerec * gmx_restrict fr,
1367 t_mdatoms * gmx_restrict mdatoms,
1368 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1369 t_nrnb * gmx_restrict nrnb)
1371 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1372 * just 0 for non-waters.
1373 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
1374 * jnr indices corresponding to data put in the four positions in the SIMD register.
1376 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1377 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1379 int j_coord_offsetA,j_coord_offsetB;
1380 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1381 real rcutoff_scalar;
1382 real *shiftvec,*fshift,*x,*f;
1383 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
1385 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1387 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1389 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1391 __m128d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
1392 int vdwjidx0A,vdwjidx0B;
1393 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1394 int vdwjidx1A,vdwjidx1B;
1395 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1396 int vdwjidx2A,vdwjidx2B;
1397 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1398 int vdwjidx3A,vdwjidx3B;
1399 __m128d jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
1400 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1401 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1402 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1403 __m128d dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
1404 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1405 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1406 __m128d dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
1407 __m128d dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
1408 __m128d dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
1409 __m128d dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
1410 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
1413 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1416 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
1417 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
1419 __m128i ifour = _mm_set1_epi32(4);
1420 __m128d rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF,twovfeps;
1423 __m128d ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
1425 __m128d dummy_mask,cutoff_mask;
1426 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
1427 __m128d one = _mm_set1_pd(1.0);
1428 __m128d two = _mm_set1_pd(2.0);
1434 jindex = nlist->jindex;
1436 shiftidx = nlist->shift;
1438 shiftvec = fr->shift_vec[0];
1439 fshift = fr->fshift[0];
1440 facel = _mm_set1_pd(fr->ic->epsfac);
1441 charge = mdatoms->chargeA;
1442 nvdwtype = fr->ntype;
1443 vdwparam = fr->nbfp;
1444 vdwtype = mdatoms->typeA;
1446 vftab = kernel_data->table_vdw->data;
1447 vftabscale = _mm_set1_pd(kernel_data->table_vdw->scale);
1449 sh_ewald = _mm_set1_pd(fr->ic->sh_ewald);
1450 ewtab = fr->ic->tabq_coul_F;
1451 ewtabscale = _mm_set1_pd(fr->ic->tabq_scale);
1452 ewtabhalfspace = _mm_set1_pd(0.5/fr->ic->tabq_scale);
1454 /* Setup water-specific parameters */
1455 inr = nlist->iinr[0];
1456 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
1457 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
1458 iq3 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+3]));
1459 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
1461 jq1 = _mm_set1_pd(charge[inr+1]);
1462 jq2 = _mm_set1_pd(charge[inr+2]);
1463 jq3 = _mm_set1_pd(charge[inr+3]);
1464 vdwjidx0A = 2*vdwtype[inr+0];
1465 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
1466 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
1467 qq11 = _mm_mul_pd(iq1,jq1);
1468 qq12 = _mm_mul_pd(iq1,jq2);
1469 qq13 = _mm_mul_pd(iq1,jq3);
1470 qq21 = _mm_mul_pd(iq2,jq1);
1471 qq22 = _mm_mul_pd(iq2,jq2);
1472 qq23 = _mm_mul_pd(iq2,jq3);
1473 qq31 = _mm_mul_pd(iq3,jq1);
1474 qq32 = _mm_mul_pd(iq3,jq2);
1475 qq33 = _mm_mul_pd(iq3,jq3);
1477 /* Avoid stupid compiler warnings */
1479 j_coord_offsetA = 0;
1480 j_coord_offsetB = 0;
1485 /* Start outer loop over neighborlists */
1486 for(iidx=0; iidx<nri; iidx++)
1488 /* Load shift vector for this list */
1489 i_shift_offset = DIM*shiftidx[iidx];
1491 /* Load limits for loop over neighbors */
1492 j_index_start = jindex[iidx];
1493 j_index_end = jindex[iidx+1];
1495 /* Get outer coordinate index */
1497 i_coord_offset = DIM*inr;
1499 /* Load i particle coords and add shift vector */
1500 gmx_mm_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
1501 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
1503 fix0 = _mm_setzero_pd();
1504 fiy0 = _mm_setzero_pd();
1505 fiz0 = _mm_setzero_pd();
1506 fix1 = _mm_setzero_pd();
1507 fiy1 = _mm_setzero_pd();
1508 fiz1 = _mm_setzero_pd();
1509 fix2 = _mm_setzero_pd();
1510 fiy2 = _mm_setzero_pd();
1511 fiz2 = _mm_setzero_pd();
1512 fix3 = _mm_setzero_pd();
1513 fiy3 = _mm_setzero_pd();
1514 fiz3 = _mm_setzero_pd();
1516 /* Start inner kernel loop */
1517 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
1520 /* Get j neighbor index, and coordinate index */
1522 jnrB = jjnr[jidx+1];
1523 j_coord_offsetA = DIM*jnrA;
1524 j_coord_offsetB = DIM*jnrB;
1526 /* load j atom coordinates */
1527 gmx_mm_load_4rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1528 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1529 &jy2,&jz2,&jx3,&jy3,&jz3);
1531 /* Calculate displacement vector */
1532 dx00 = _mm_sub_pd(ix0,jx0);
1533 dy00 = _mm_sub_pd(iy0,jy0);
1534 dz00 = _mm_sub_pd(iz0,jz0);
1535 dx11 = _mm_sub_pd(ix1,jx1);
1536 dy11 = _mm_sub_pd(iy1,jy1);
1537 dz11 = _mm_sub_pd(iz1,jz1);
1538 dx12 = _mm_sub_pd(ix1,jx2);
1539 dy12 = _mm_sub_pd(iy1,jy2);
1540 dz12 = _mm_sub_pd(iz1,jz2);
1541 dx13 = _mm_sub_pd(ix1,jx3);
1542 dy13 = _mm_sub_pd(iy1,jy3);
1543 dz13 = _mm_sub_pd(iz1,jz3);
1544 dx21 = _mm_sub_pd(ix2,jx1);
1545 dy21 = _mm_sub_pd(iy2,jy1);
1546 dz21 = _mm_sub_pd(iz2,jz1);
1547 dx22 = _mm_sub_pd(ix2,jx2);
1548 dy22 = _mm_sub_pd(iy2,jy2);
1549 dz22 = _mm_sub_pd(iz2,jz2);
1550 dx23 = _mm_sub_pd(ix2,jx3);
1551 dy23 = _mm_sub_pd(iy2,jy3);
1552 dz23 = _mm_sub_pd(iz2,jz3);
1553 dx31 = _mm_sub_pd(ix3,jx1);
1554 dy31 = _mm_sub_pd(iy3,jy1);
1555 dz31 = _mm_sub_pd(iz3,jz1);
1556 dx32 = _mm_sub_pd(ix3,jx2);
1557 dy32 = _mm_sub_pd(iy3,jy2);
1558 dz32 = _mm_sub_pd(iz3,jz2);
1559 dx33 = _mm_sub_pd(ix3,jx3);
1560 dy33 = _mm_sub_pd(iy3,jy3);
1561 dz33 = _mm_sub_pd(iz3,jz3);
1563 /* Calculate squared distance and things based on it */
1564 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1565 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1566 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1567 rsq13 = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
1568 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1569 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1570 rsq23 = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
1571 rsq31 = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
1572 rsq32 = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
1573 rsq33 = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
1575 rinv00 = avx128fma_invsqrt_d(rsq00);
1576 rinv11 = avx128fma_invsqrt_d(rsq11);
1577 rinv12 = avx128fma_invsqrt_d(rsq12);
1578 rinv13 = avx128fma_invsqrt_d(rsq13);
1579 rinv21 = avx128fma_invsqrt_d(rsq21);
1580 rinv22 = avx128fma_invsqrt_d(rsq22);
1581 rinv23 = avx128fma_invsqrt_d(rsq23);
1582 rinv31 = avx128fma_invsqrt_d(rsq31);
1583 rinv32 = avx128fma_invsqrt_d(rsq32);
1584 rinv33 = avx128fma_invsqrt_d(rsq33);
1586 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
1587 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
1588 rinvsq13 = _mm_mul_pd(rinv13,rinv13);
1589 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
1590 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
1591 rinvsq23 = _mm_mul_pd(rinv23,rinv23);
1592 rinvsq31 = _mm_mul_pd(rinv31,rinv31);
1593 rinvsq32 = _mm_mul_pd(rinv32,rinv32);
1594 rinvsq33 = _mm_mul_pd(rinv33,rinv33);
1596 fjx0 = _mm_setzero_pd();
1597 fjy0 = _mm_setzero_pd();
1598 fjz0 = _mm_setzero_pd();
1599 fjx1 = _mm_setzero_pd();
1600 fjy1 = _mm_setzero_pd();
1601 fjz1 = _mm_setzero_pd();
1602 fjx2 = _mm_setzero_pd();
1603 fjy2 = _mm_setzero_pd();
1604 fjz2 = _mm_setzero_pd();
1605 fjx3 = _mm_setzero_pd();
1606 fjy3 = _mm_setzero_pd();
1607 fjz3 = _mm_setzero_pd();
1609 /**************************
1610 * CALCULATE INTERACTIONS *
1611 **************************/
1613 r00 = _mm_mul_pd(rsq00,rinv00);
1615 /* Calculate table index by multiplying r with table scale and truncate to integer */
1616 rt = _mm_mul_pd(r00,vftabscale);
1617 vfitab = _mm_cvttpd_epi32(rt);
1619 vfeps = _mm_frcz_pd(rt);
1621 vfeps = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
1623 twovfeps = _mm_add_pd(vfeps,vfeps);
1624 vfitab = _mm_slli_epi32(vfitab,3);
1626 /* CUBIC SPLINE TABLE DISPERSION */
1627 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1628 F = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1629 GMX_MM_TRANSPOSE2_PD(Y,F);
1630 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1631 H = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
1632 GMX_MM_TRANSPOSE2_PD(G,H);
1633 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1634 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1635 fvdw6 = _mm_mul_pd(c6_00,FF);
1637 /* CUBIC SPLINE TABLE REPULSION */
1638 vfitab = _mm_add_epi32(vfitab,ifour);
1639 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1640 F = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1641 GMX_MM_TRANSPOSE2_PD(Y,F);
1642 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
1643 H = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,1) +2);
1644 GMX_MM_TRANSPOSE2_PD(G,H);
1645 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
1646 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
1647 fvdw12 = _mm_mul_pd(c12_00,FF);
1648 fvdw = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
1652 /* Update vectorial force */
1653 fix0 = _mm_macc_pd(dx00,fscal,fix0);
1654 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
1655 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
1657 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
1658 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
1659 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
1661 /**************************
1662 * CALCULATE INTERACTIONS *
1663 **************************/
1665 r11 = _mm_mul_pd(rsq11,rinv11);
1667 /* EWALD ELECTROSTATICS */
1669 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1670 ewrt = _mm_mul_pd(r11,ewtabscale);
1671 ewitab = _mm_cvttpd_epi32(ewrt);
1673 eweps = _mm_frcz_pd(ewrt);
1675 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1677 twoeweps = _mm_add_pd(eweps,eweps);
1678 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1680 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1681 felec = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
1685 /* Update vectorial force */
1686 fix1 = _mm_macc_pd(dx11,fscal,fix1);
1687 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
1688 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
1690 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
1691 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
1692 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
1694 /**************************
1695 * CALCULATE INTERACTIONS *
1696 **************************/
1698 r12 = _mm_mul_pd(rsq12,rinv12);
1700 /* EWALD ELECTROSTATICS */
1702 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1703 ewrt = _mm_mul_pd(r12,ewtabscale);
1704 ewitab = _mm_cvttpd_epi32(ewrt);
1706 eweps = _mm_frcz_pd(ewrt);
1708 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1710 twoeweps = _mm_add_pd(eweps,eweps);
1711 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1713 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1714 felec = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
1718 /* Update vectorial force */
1719 fix1 = _mm_macc_pd(dx12,fscal,fix1);
1720 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
1721 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1723 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1724 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1725 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1727 /**************************
1728 * CALCULATE INTERACTIONS *
1729 **************************/
1731 r13 = _mm_mul_pd(rsq13,rinv13);
1733 /* EWALD ELECTROSTATICS */
1735 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1736 ewrt = _mm_mul_pd(r13,ewtabscale);
1737 ewitab = _mm_cvttpd_epi32(ewrt);
1739 eweps = _mm_frcz_pd(ewrt);
1741 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1743 twoeweps = _mm_add_pd(eweps,eweps);
1744 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1746 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1747 felec = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
1751 /* Update vectorial force */
1752 fix1 = _mm_macc_pd(dx13,fscal,fix1);
1753 fiy1 = _mm_macc_pd(dy13,fscal,fiy1);
1754 fiz1 = _mm_macc_pd(dz13,fscal,fiz1);
1756 fjx3 = _mm_macc_pd(dx13,fscal,fjx3);
1757 fjy3 = _mm_macc_pd(dy13,fscal,fjy3);
1758 fjz3 = _mm_macc_pd(dz13,fscal,fjz3);
1760 /**************************
1761 * CALCULATE INTERACTIONS *
1762 **************************/
1764 r21 = _mm_mul_pd(rsq21,rinv21);
1766 /* EWALD ELECTROSTATICS */
1768 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1769 ewrt = _mm_mul_pd(r21,ewtabscale);
1770 ewitab = _mm_cvttpd_epi32(ewrt);
1772 eweps = _mm_frcz_pd(ewrt);
1774 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1776 twoeweps = _mm_add_pd(eweps,eweps);
1777 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1779 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1780 felec = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
1784 /* Update vectorial force */
1785 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1786 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1787 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1789 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1790 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1791 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1793 /**************************
1794 * CALCULATE INTERACTIONS *
1795 **************************/
1797 r22 = _mm_mul_pd(rsq22,rinv22);
1799 /* EWALD ELECTROSTATICS */
1801 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1802 ewrt = _mm_mul_pd(r22,ewtabscale);
1803 ewitab = _mm_cvttpd_epi32(ewrt);
1805 eweps = _mm_frcz_pd(ewrt);
1807 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1809 twoeweps = _mm_add_pd(eweps,eweps);
1810 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1812 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1813 felec = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
1817 /* Update vectorial force */
1818 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1819 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1820 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1822 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1823 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1824 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1826 /**************************
1827 * CALCULATE INTERACTIONS *
1828 **************************/
1830 r23 = _mm_mul_pd(rsq23,rinv23);
1832 /* EWALD ELECTROSTATICS */
1834 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1835 ewrt = _mm_mul_pd(r23,ewtabscale);
1836 ewitab = _mm_cvttpd_epi32(ewrt);
1838 eweps = _mm_frcz_pd(ewrt);
1840 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1842 twoeweps = _mm_add_pd(eweps,eweps);
1843 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1845 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1846 felec = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
1850 /* Update vectorial force */
1851 fix2 = _mm_macc_pd(dx23,fscal,fix2);
1852 fiy2 = _mm_macc_pd(dy23,fscal,fiy2);
1853 fiz2 = _mm_macc_pd(dz23,fscal,fiz2);
1855 fjx3 = _mm_macc_pd(dx23,fscal,fjx3);
1856 fjy3 = _mm_macc_pd(dy23,fscal,fjy3);
1857 fjz3 = _mm_macc_pd(dz23,fscal,fjz3);
1859 /**************************
1860 * CALCULATE INTERACTIONS *
1861 **************************/
1863 r31 = _mm_mul_pd(rsq31,rinv31);
1865 /* EWALD ELECTROSTATICS */
1867 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1868 ewrt = _mm_mul_pd(r31,ewtabscale);
1869 ewitab = _mm_cvttpd_epi32(ewrt);
1871 eweps = _mm_frcz_pd(ewrt);
1873 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1875 twoeweps = _mm_add_pd(eweps,eweps);
1876 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1878 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1879 felec = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
1883 /* Update vectorial force */
1884 fix3 = _mm_macc_pd(dx31,fscal,fix3);
1885 fiy3 = _mm_macc_pd(dy31,fscal,fiy3);
1886 fiz3 = _mm_macc_pd(dz31,fscal,fiz3);
1888 fjx1 = _mm_macc_pd(dx31,fscal,fjx1);
1889 fjy1 = _mm_macc_pd(dy31,fscal,fjy1);
1890 fjz1 = _mm_macc_pd(dz31,fscal,fjz1);
1892 /**************************
1893 * CALCULATE INTERACTIONS *
1894 **************************/
1896 r32 = _mm_mul_pd(rsq32,rinv32);
1898 /* EWALD ELECTROSTATICS */
1900 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1901 ewrt = _mm_mul_pd(r32,ewtabscale);
1902 ewitab = _mm_cvttpd_epi32(ewrt);
1904 eweps = _mm_frcz_pd(ewrt);
1906 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1908 twoeweps = _mm_add_pd(eweps,eweps);
1909 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1911 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1912 felec = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
1916 /* Update vectorial force */
1917 fix3 = _mm_macc_pd(dx32,fscal,fix3);
1918 fiy3 = _mm_macc_pd(dy32,fscal,fiy3);
1919 fiz3 = _mm_macc_pd(dz32,fscal,fiz3);
1921 fjx2 = _mm_macc_pd(dx32,fscal,fjx2);
1922 fjy2 = _mm_macc_pd(dy32,fscal,fjy2);
1923 fjz2 = _mm_macc_pd(dz32,fscal,fjz2);
1925 /**************************
1926 * CALCULATE INTERACTIONS *
1927 **************************/
1929 r33 = _mm_mul_pd(rsq33,rinv33);
1931 /* EWALD ELECTROSTATICS */
1933 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
1934 ewrt = _mm_mul_pd(r33,ewtabscale);
1935 ewitab = _mm_cvttpd_epi32(ewrt);
1937 eweps = _mm_frcz_pd(ewrt);
1939 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
1941 twoeweps = _mm_add_pd(eweps,eweps);
1942 gmx_mm_load_2pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),ewtab+_mm_extract_epi32(ewitab,1),
1944 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
1945 felec = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
1949 /* Update vectorial force */
1950 fix3 = _mm_macc_pd(dx33,fscal,fix3);
1951 fiy3 = _mm_macc_pd(dy33,fscal,fiy3);
1952 fiz3 = _mm_macc_pd(dz33,fscal,fiz3);
1954 fjx3 = _mm_macc_pd(dx33,fscal,fjx3);
1955 fjy3 = _mm_macc_pd(dy33,fscal,fjy3);
1956 fjz3 = _mm_macc_pd(dz33,fscal,fjz3);
1958 gmx_mm_decrement_4rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1960 /* Inner loop uses 405 flops */
1963 if(jidx<j_index_end)
1967 j_coord_offsetA = DIM*jnrA;
1969 /* load j atom coordinates */
1970 gmx_mm_load_4rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
1971 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1972 &jy2,&jz2,&jx3,&jy3,&jz3);
1974 /* Calculate displacement vector */
1975 dx00 = _mm_sub_pd(ix0,jx0);
1976 dy00 = _mm_sub_pd(iy0,jy0);
1977 dz00 = _mm_sub_pd(iz0,jz0);
1978 dx11 = _mm_sub_pd(ix1,jx1);
1979 dy11 = _mm_sub_pd(iy1,jy1);
1980 dz11 = _mm_sub_pd(iz1,jz1);
1981 dx12 = _mm_sub_pd(ix1,jx2);
1982 dy12 = _mm_sub_pd(iy1,jy2);
1983 dz12 = _mm_sub_pd(iz1,jz2);
1984 dx13 = _mm_sub_pd(ix1,jx3);
1985 dy13 = _mm_sub_pd(iy1,jy3);
1986 dz13 = _mm_sub_pd(iz1,jz3);
1987 dx21 = _mm_sub_pd(ix2,jx1);
1988 dy21 = _mm_sub_pd(iy2,jy1);
1989 dz21 = _mm_sub_pd(iz2,jz1);
1990 dx22 = _mm_sub_pd(ix2,jx2);
1991 dy22 = _mm_sub_pd(iy2,jy2);
1992 dz22 = _mm_sub_pd(iz2,jz2);
1993 dx23 = _mm_sub_pd(ix2,jx3);
1994 dy23 = _mm_sub_pd(iy2,jy3);
1995 dz23 = _mm_sub_pd(iz2,jz3);
1996 dx31 = _mm_sub_pd(ix3,jx1);
1997 dy31 = _mm_sub_pd(iy3,jy1);
1998 dz31 = _mm_sub_pd(iz3,jz1);
1999 dx32 = _mm_sub_pd(ix3,jx2);
2000 dy32 = _mm_sub_pd(iy3,jy2);
2001 dz32 = _mm_sub_pd(iz3,jz2);
2002 dx33 = _mm_sub_pd(ix3,jx3);
2003 dy33 = _mm_sub_pd(iy3,jy3);
2004 dz33 = _mm_sub_pd(iz3,jz3);
2006 /* Calculate squared distance and things based on it */
2007 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
2008 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
2009 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
2010 rsq13 = gmx_mm_calc_rsq_pd(dx13,dy13,dz13);
2011 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
2012 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
2013 rsq23 = gmx_mm_calc_rsq_pd(dx23,dy23,dz23);
2014 rsq31 = gmx_mm_calc_rsq_pd(dx31,dy31,dz31);
2015 rsq32 = gmx_mm_calc_rsq_pd(dx32,dy32,dz32);
2016 rsq33 = gmx_mm_calc_rsq_pd(dx33,dy33,dz33);
2018 rinv00 = avx128fma_invsqrt_d(rsq00);
2019 rinv11 = avx128fma_invsqrt_d(rsq11);
2020 rinv12 = avx128fma_invsqrt_d(rsq12);
2021 rinv13 = avx128fma_invsqrt_d(rsq13);
2022 rinv21 = avx128fma_invsqrt_d(rsq21);
2023 rinv22 = avx128fma_invsqrt_d(rsq22);
2024 rinv23 = avx128fma_invsqrt_d(rsq23);
2025 rinv31 = avx128fma_invsqrt_d(rsq31);
2026 rinv32 = avx128fma_invsqrt_d(rsq32);
2027 rinv33 = avx128fma_invsqrt_d(rsq33);
2029 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
2030 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
2031 rinvsq13 = _mm_mul_pd(rinv13,rinv13);
2032 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
2033 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
2034 rinvsq23 = _mm_mul_pd(rinv23,rinv23);
2035 rinvsq31 = _mm_mul_pd(rinv31,rinv31);
2036 rinvsq32 = _mm_mul_pd(rinv32,rinv32);
2037 rinvsq33 = _mm_mul_pd(rinv33,rinv33);
2039 fjx0 = _mm_setzero_pd();
2040 fjy0 = _mm_setzero_pd();
2041 fjz0 = _mm_setzero_pd();
2042 fjx1 = _mm_setzero_pd();
2043 fjy1 = _mm_setzero_pd();
2044 fjz1 = _mm_setzero_pd();
2045 fjx2 = _mm_setzero_pd();
2046 fjy2 = _mm_setzero_pd();
2047 fjz2 = _mm_setzero_pd();
2048 fjx3 = _mm_setzero_pd();
2049 fjy3 = _mm_setzero_pd();
2050 fjz3 = _mm_setzero_pd();
2052 /**************************
2053 * CALCULATE INTERACTIONS *
2054 **************************/
2056 r00 = _mm_mul_pd(rsq00,rinv00);
2058 /* Calculate table index by multiplying r with table scale and truncate to integer */
2059 rt = _mm_mul_pd(r00,vftabscale);
2060 vfitab = _mm_cvttpd_epi32(rt);
2062 vfeps = _mm_frcz_pd(rt);
2064 vfeps = _mm_sub_pd(rt,_mm_round_pd(rt, _MM_FROUND_FLOOR));
2066 twovfeps = _mm_add_pd(vfeps,vfeps);
2067 vfitab = _mm_slli_epi32(vfitab,3);
2069 /* CUBIC SPLINE TABLE DISPERSION */
2070 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
2071 F = _mm_setzero_pd();
2072 GMX_MM_TRANSPOSE2_PD(Y,F);
2073 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
2074 H = _mm_setzero_pd();
2075 GMX_MM_TRANSPOSE2_PD(G,H);
2076 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
2077 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
2078 fvdw6 = _mm_mul_pd(c6_00,FF);
2080 /* CUBIC SPLINE TABLE REPULSION */
2081 vfitab = _mm_add_epi32(vfitab,ifour);
2082 Y = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
2083 F = _mm_setzero_pd();
2084 GMX_MM_TRANSPOSE2_PD(Y,F);
2085 G = _mm_load_pd( vftab + _mm_extract_epi32(vfitab,0) +2);
2086 H = _mm_setzero_pd();
2087 GMX_MM_TRANSPOSE2_PD(G,H);
2088 Fp = _mm_macc_pd(vfeps,_mm_macc_pd(H,vfeps,G),F);
2089 FF = _mm_macc_pd(vfeps,_mm_macc_pd(twovfeps,H,G),Fp);
2090 fvdw12 = _mm_mul_pd(c12_00,FF);
2091 fvdw = _mm_xor_pd(signbit,_mm_mul_pd(_mm_add_pd(fvdw6,fvdw12),_mm_mul_pd(vftabscale,rinv00)));
2095 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2097 /* Update vectorial force */
2098 fix0 = _mm_macc_pd(dx00,fscal,fix0);
2099 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
2100 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
2102 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
2103 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
2104 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
2106 /**************************
2107 * CALCULATE INTERACTIONS *
2108 **************************/
2110 r11 = _mm_mul_pd(rsq11,rinv11);
2112 /* EWALD ELECTROSTATICS */
2114 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2115 ewrt = _mm_mul_pd(r11,ewtabscale);
2116 ewitab = _mm_cvttpd_epi32(ewrt);
2118 eweps = _mm_frcz_pd(ewrt);
2120 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2122 twoeweps = _mm_add_pd(eweps,eweps);
2123 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2124 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2125 felec = _mm_mul_pd(_mm_mul_pd(qq11,rinv11),_mm_sub_pd(rinvsq11,felec));
2129 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2131 /* Update vectorial force */
2132 fix1 = _mm_macc_pd(dx11,fscal,fix1);
2133 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
2134 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
2136 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
2137 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
2138 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
2140 /**************************
2141 * CALCULATE INTERACTIONS *
2142 **************************/
2144 r12 = _mm_mul_pd(rsq12,rinv12);
2146 /* EWALD ELECTROSTATICS */
2148 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2149 ewrt = _mm_mul_pd(r12,ewtabscale);
2150 ewitab = _mm_cvttpd_epi32(ewrt);
2152 eweps = _mm_frcz_pd(ewrt);
2154 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2156 twoeweps = _mm_add_pd(eweps,eweps);
2157 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2158 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2159 felec = _mm_mul_pd(_mm_mul_pd(qq12,rinv12),_mm_sub_pd(rinvsq12,felec));
2163 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2165 /* Update vectorial force */
2166 fix1 = _mm_macc_pd(dx12,fscal,fix1);
2167 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
2168 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
2170 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
2171 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
2172 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
2174 /**************************
2175 * CALCULATE INTERACTIONS *
2176 **************************/
2178 r13 = _mm_mul_pd(rsq13,rinv13);
2180 /* EWALD ELECTROSTATICS */
2182 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2183 ewrt = _mm_mul_pd(r13,ewtabscale);
2184 ewitab = _mm_cvttpd_epi32(ewrt);
2186 eweps = _mm_frcz_pd(ewrt);
2188 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2190 twoeweps = _mm_add_pd(eweps,eweps);
2191 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2192 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2193 felec = _mm_mul_pd(_mm_mul_pd(qq13,rinv13),_mm_sub_pd(rinvsq13,felec));
2197 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2199 /* Update vectorial force */
2200 fix1 = _mm_macc_pd(dx13,fscal,fix1);
2201 fiy1 = _mm_macc_pd(dy13,fscal,fiy1);
2202 fiz1 = _mm_macc_pd(dz13,fscal,fiz1);
2204 fjx3 = _mm_macc_pd(dx13,fscal,fjx3);
2205 fjy3 = _mm_macc_pd(dy13,fscal,fjy3);
2206 fjz3 = _mm_macc_pd(dz13,fscal,fjz3);
2208 /**************************
2209 * CALCULATE INTERACTIONS *
2210 **************************/
2212 r21 = _mm_mul_pd(rsq21,rinv21);
2214 /* EWALD ELECTROSTATICS */
2216 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2217 ewrt = _mm_mul_pd(r21,ewtabscale);
2218 ewitab = _mm_cvttpd_epi32(ewrt);
2220 eweps = _mm_frcz_pd(ewrt);
2222 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2224 twoeweps = _mm_add_pd(eweps,eweps);
2225 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2226 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2227 felec = _mm_mul_pd(_mm_mul_pd(qq21,rinv21),_mm_sub_pd(rinvsq21,felec));
2231 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2233 /* Update vectorial force */
2234 fix2 = _mm_macc_pd(dx21,fscal,fix2);
2235 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
2236 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
2238 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
2239 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
2240 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
2242 /**************************
2243 * CALCULATE INTERACTIONS *
2244 **************************/
2246 r22 = _mm_mul_pd(rsq22,rinv22);
2248 /* EWALD ELECTROSTATICS */
2250 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2251 ewrt = _mm_mul_pd(r22,ewtabscale);
2252 ewitab = _mm_cvttpd_epi32(ewrt);
2254 eweps = _mm_frcz_pd(ewrt);
2256 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2258 twoeweps = _mm_add_pd(eweps,eweps);
2259 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2260 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2261 felec = _mm_mul_pd(_mm_mul_pd(qq22,rinv22),_mm_sub_pd(rinvsq22,felec));
2265 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2267 /* Update vectorial force */
2268 fix2 = _mm_macc_pd(dx22,fscal,fix2);
2269 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
2270 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
2272 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
2273 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
2274 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
2276 /**************************
2277 * CALCULATE INTERACTIONS *
2278 **************************/
2280 r23 = _mm_mul_pd(rsq23,rinv23);
2282 /* EWALD ELECTROSTATICS */
2284 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2285 ewrt = _mm_mul_pd(r23,ewtabscale);
2286 ewitab = _mm_cvttpd_epi32(ewrt);
2288 eweps = _mm_frcz_pd(ewrt);
2290 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2292 twoeweps = _mm_add_pd(eweps,eweps);
2293 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2294 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2295 felec = _mm_mul_pd(_mm_mul_pd(qq23,rinv23),_mm_sub_pd(rinvsq23,felec));
2299 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2301 /* Update vectorial force */
2302 fix2 = _mm_macc_pd(dx23,fscal,fix2);
2303 fiy2 = _mm_macc_pd(dy23,fscal,fiy2);
2304 fiz2 = _mm_macc_pd(dz23,fscal,fiz2);
2306 fjx3 = _mm_macc_pd(dx23,fscal,fjx3);
2307 fjy3 = _mm_macc_pd(dy23,fscal,fjy3);
2308 fjz3 = _mm_macc_pd(dz23,fscal,fjz3);
2310 /**************************
2311 * CALCULATE INTERACTIONS *
2312 **************************/
2314 r31 = _mm_mul_pd(rsq31,rinv31);
2316 /* EWALD ELECTROSTATICS */
2318 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2319 ewrt = _mm_mul_pd(r31,ewtabscale);
2320 ewitab = _mm_cvttpd_epi32(ewrt);
2322 eweps = _mm_frcz_pd(ewrt);
2324 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2326 twoeweps = _mm_add_pd(eweps,eweps);
2327 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2328 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2329 felec = _mm_mul_pd(_mm_mul_pd(qq31,rinv31),_mm_sub_pd(rinvsq31,felec));
2333 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2335 /* Update vectorial force */
2336 fix3 = _mm_macc_pd(dx31,fscal,fix3);
2337 fiy3 = _mm_macc_pd(dy31,fscal,fiy3);
2338 fiz3 = _mm_macc_pd(dz31,fscal,fiz3);
2340 fjx1 = _mm_macc_pd(dx31,fscal,fjx1);
2341 fjy1 = _mm_macc_pd(dy31,fscal,fjy1);
2342 fjz1 = _mm_macc_pd(dz31,fscal,fjz1);
2344 /**************************
2345 * CALCULATE INTERACTIONS *
2346 **************************/
2348 r32 = _mm_mul_pd(rsq32,rinv32);
2350 /* EWALD ELECTROSTATICS */
2352 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2353 ewrt = _mm_mul_pd(r32,ewtabscale);
2354 ewitab = _mm_cvttpd_epi32(ewrt);
2356 eweps = _mm_frcz_pd(ewrt);
2358 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2360 twoeweps = _mm_add_pd(eweps,eweps);
2361 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2362 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2363 felec = _mm_mul_pd(_mm_mul_pd(qq32,rinv32),_mm_sub_pd(rinvsq32,felec));
2367 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2369 /* Update vectorial force */
2370 fix3 = _mm_macc_pd(dx32,fscal,fix3);
2371 fiy3 = _mm_macc_pd(dy32,fscal,fiy3);
2372 fiz3 = _mm_macc_pd(dz32,fscal,fiz3);
2374 fjx2 = _mm_macc_pd(dx32,fscal,fjx2);
2375 fjy2 = _mm_macc_pd(dy32,fscal,fjy2);
2376 fjz2 = _mm_macc_pd(dz32,fscal,fjz2);
2378 /**************************
2379 * CALCULATE INTERACTIONS *
2380 **************************/
2382 r33 = _mm_mul_pd(rsq33,rinv33);
2384 /* EWALD ELECTROSTATICS */
2386 /* Calculate Ewald table index by multiplying r with scale and truncate to integer */
2387 ewrt = _mm_mul_pd(r33,ewtabscale);
2388 ewitab = _mm_cvttpd_epi32(ewrt);
2390 eweps = _mm_frcz_pd(ewrt);
2392 eweps = _mm_sub_pd(ewrt,_mm_round_pd(ewrt, _MM_FROUND_FLOOR));
2394 twoeweps = _mm_add_pd(eweps,eweps);
2395 gmx_mm_load_1pair_swizzle_pd(ewtab+_mm_extract_epi32(ewitab,0),&ewtabF,&ewtabFn);
2396 felec = _mm_macc_pd(eweps,ewtabFn,_mm_mul_pd( _mm_sub_pd(one,eweps),ewtabF));
2397 felec = _mm_mul_pd(_mm_mul_pd(qq33,rinv33),_mm_sub_pd(rinvsq33,felec));
2401 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
2403 /* Update vectorial force */
2404 fix3 = _mm_macc_pd(dx33,fscal,fix3);
2405 fiy3 = _mm_macc_pd(dy33,fscal,fiy3);
2406 fiz3 = _mm_macc_pd(dz33,fscal,fiz3);
2408 fjx3 = _mm_macc_pd(dx33,fscal,fjx3);
2409 fjy3 = _mm_macc_pd(dy33,fscal,fjy3);
2410 fjz3 = _mm_macc_pd(dz33,fscal,fjz3);
2412 gmx_mm_decrement_4rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
2414 /* Inner loop uses 405 flops */
2417 /* End of innermost loop */
2419 gmx_mm_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
2420 f+i_coord_offset,fshift+i_shift_offset);
2422 /* Increment number of inner iterations */
2423 inneriter += j_index_end - j_index_start;
2425 /* Outer loop uses 24 flops */
2428 /* Increment number of outer iterations */
2431 /* Update outer/inner flops */
2433 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_F,outeriter*24 + inneriter*405);