2 * Note: this file was generated by the Gromacs avx_128_fma_double kernel generator.
4 * This source code is part of
8 * Copyright (c) 2001-2012, The GROMACS Development Team
10 * Gromacs is a library for molecular simulation and trajectory analysis,
11 * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
12 * a full list of developers and information, check out http://www.gromacs.org
14 * This program is free software; you can redistribute it and/or modify it under
15 * the terms of the GNU Lesser General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option) any
19 * To help fund GROMACS development, we humbly ask that you cite
20 * the papers people have written on it - you can find them on the website.
28 #include "../nb_kernel.h"
29 #include "types/simple.h"
33 #include "gmx_math_x86_avx_128_fma_double.h"
34 #include "kernelutil_x86_avx_128_fma_double.h"
37 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwLJSw_GeomW3W3_VF_avx_128_fma_double
38 * Electrostatics interaction: ReactionField
39 * VdW interaction: LennardJones
40 * Geometry: Water3-Water3
41 * Calculate force/pot: PotentialAndForce
44 nb_kernel_ElecRFCut_VdwLJSw_GeomW3W3_VF_avx_128_fma_double
45 (t_nblist * gmx_restrict nlist,
46 rvec * gmx_restrict xx,
47 rvec * gmx_restrict ff,
48 t_forcerec * gmx_restrict fr,
49 t_mdatoms * gmx_restrict mdatoms,
50 nb_kernel_data_t * gmx_restrict kernel_data,
51 t_nrnb * gmx_restrict nrnb)
53 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
54 * just 0 for non-waters.
55 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
56 * jnr indices corresponding to data put in the four positions in the SIMD register.
58 int i_shift_offset,i_coord_offset,outeriter,inneriter;
59 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
61 int j_coord_offsetA,j_coord_offsetB;
62 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
64 real *shiftvec,*fshift,*x,*f;
65 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
67 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
69 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
71 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
72 int vdwjidx0A,vdwjidx0B;
73 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
74 int vdwjidx1A,vdwjidx1B;
75 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
76 int vdwjidx2A,vdwjidx2B;
77 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
78 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
79 __m128d dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
80 __m128d dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
81 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
82 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
83 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
84 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
85 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
86 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
87 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
90 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
93 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
94 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
95 __m128d rswitch,swV3,swV4,swV5,swF2,swF3,swF4,d,d2,sw,dsw;
96 real rswitch_scalar,d_scalar;
97 __m128d dummy_mask,cutoff_mask;
98 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
99 __m128d one = _mm_set1_pd(1.0);
100 __m128d two = _mm_set1_pd(2.0);
106 jindex = nlist->jindex;
108 shiftidx = nlist->shift;
110 shiftvec = fr->shift_vec[0];
111 fshift = fr->fshift[0];
112 facel = _mm_set1_pd(fr->epsfac);
113 charge = mdatoms->chargeA;
114 krf = _mm_set1_pd(fr->ic->k_rf);
115 krf2 = _mm_set1_pd(fr->ic->k_rf*2.0);
116 crf = _mm_set1_pd(fr->ic->c_rf);
117 nvdwtype = fr->ntype;
119 vdwtype = mdatoms->typeA;
121 /* Setup water-specific parameters */
122 inr = nlist->iinr[0];
123 iq0 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
124 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
125 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
126 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
128 jq0 = _mm_set1_pd(charge[inr+0]);
129 jq1 = _mm_set1_pd(charge[inr+1]);
130 jq2 = _mm_set1_pd(charge[inr+2]);
131 vdwjidx0A = 2*vdwtype[inr+0];
132 qq00 = _mm_mul_pd(iq0,jq0);
133 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
134 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
135 qq01 = _mm_mul_pd(iq0,jq1);
136 qq02 = _mm_mul_pd(iq0,jq2);
137 qq10 = _mm_mul_pd(iq1,jq0);
138 qq11 = _mm_mul_pd(iq1,jq1);
139 qq12 = _mm_mul_pd(iq1,jq2);
140 qq20 = _mm_mul_pd(iq2,jq0);
141 qq21 = _mm_mul_pd(iq2,jq1);
142 qq22 = _mm_mul_pd(iq2,jq2);
144 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
145 rcutoff_scalar = fr->rcoulomb;
146 rcutoff = _mm_set1_pd(rcutoff_scalar);
147 rcutoff2 = _mm_mul_pd(rcutoff,rcutoff);
149 rswitch_scalar = fr->rvdw_switch;
150 rswitch = _mm_set1_pd(rswitch_scalar);
151 /* Setup switch parameters */
152 d_scalar = rcutoff_scalar-rswitch_scalar;
153 d = _mm_set1_pd(d_scalar);
154 swV3 = _mm_set1_pd(-10.0/(d_scalar*d_scalar*d_scalar));
155 swV4 = _mm_set1_pd( 15.0/(d_scalar*d_scalar*d_scalar*d_scalar));
156 swV5 = _mm_set1_pd( -6.0/(d_scalar*d_scalar*d_scalar*d_scalar*d_scalar));
157 swF2 = _mm_set1_pd(-30.0/(d_scalar*d_scalar*d_scalar));
158 swF3 = _mm_set1_pd( 60.0/(d_scalar*d_scalar*d_scalar*d_scalar));
159 swF4 = _mm_set1_pd(-30.0/(d_scalar*d_scalar*d_scalar*d_scalar*d_scalar));
161 /* Avoid stupid compiler warnings */
169 /* Start outer loop over neighborlists */
170 for(iidx=0; iidx<nri; iidx++)
172 /* Load shift vector for this list */
173 i_shift_offset = DIM*shiftidx[iidx];
175 /* Load limits for loop over neighbors */
176 j_index_start = jindex[iidx];
177 j_index_end = jindex[iidx+1];
179 /* Get outer coordinate index */
181 i_coord_offset = DIM*inr;
183 /* Load i particle coords and add shift vector */
184 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
185 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
187 fix0 = _mm_setzero_pd();
188 fiy0 = _mm_setzero_pd();
189 fiz0 = _mm_setzero_pd();
190 fix1 = _mm_setzero_pd();
191 fiy1 = _mm_setzero_pd();
192 fiz1 = _mm_setzero_pd();
193 fix2 = _mm_setzero_pd();
194 fiy2 = _mm_setzero_pd();
195 fiz2 = _mm_setzero_pd();
197 /* Reset potential sums */
198 velecsum = _mm_setzero_pd();
199 vvdwsum = _mm_setzero_pd();
201 /* Start inner kernel loop */
202 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
205 /* Get j neighbor index, and coordinate index */
208 j_coord_offsetA = DIM*jnrA;
209 j_coord_offsetB = DIM*jnrB;
211 /* load j atom coordinates */
212 gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
213 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
215 /* Calculate displacement vector */
216 dx00 = _mm_sub_pd(ix0,jx0);
217 dy00 = _mm_sub_pd(iy0,jy0);
218 dz00 = _mm_sub_pd(iz0,jz0);
219 dx01 = _mm_sub_pd(ix0,jx1);
220 dy01 = _mm_sub_pd(iy0,jy1);
221 dz01 = _mm_sub_pd(iz0,jz1);
222 dx02 = _mm_sub_pd(ix0,jx2);
223 dy02 = _mm_sub_pd(iy0,jy2);
224 dz02 = _mm_sub_pd(iz0,jz2);
225 dx10 = _mm_sub_pd(ix1,jx0);
226 dy10 = _mm_sub_pd(iy1,jy0);
227 dz10 = _mm_sub_pd(iz1,jz0);
228 dx11 = _mm_sub_pd(ix1,jx1);
229 dy11 = _mm_sub_pd(iy1,jy1);
230 dz11 = _mm_sub_pd(iz1,jz1);
231 dx12 = _mm_sub_pd(ix1,jx2);
232 dy12 = _mm_sub_pd(iy1,jy2);
233 dz12 = _mm_sub_pd(iz1,jz2);
234 dx20 = _mm_sub_pd(ix2,jx0);
235 dy20 = _mm_sub_pd(iy2,jy0);
236 dz20 = _mm_sub_pd(iz2,jz0);
237 dx21 = _mm_sub_pd(ix2,jx1);
238 dy21 = _mm_sub_pd(iy2,jy1);
239 dz21 = _mm_sub_pd(iz2,jz1);
240 dx22 = _mm_sub_pd(ix2,jx2);
241 dy22 = _mm_sub_pd(iy2,jy2);
242 dz22 = _mm_sub_pd(iz2,jz2);
244 /* Calculate squared distance and things based on it */
245 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
246 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
247 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
248 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
249 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
250 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
251 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
252 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
253 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
255 rinv00 = gmx_mm_invsqrt_pd(rsq00);
256 rinv01 = gmx_mm_invsqrt_pd(rsq01);
257 rinv02 = gmx_mm_invsqrt_pd(rsq02);
258 rinv10 = gmx_mm_invsqrt_pd(rsq10);
259 rinv11 = gmx_mm_invsqrt_pd(rsq11);
260 rinv12 = gmx_mm_invsqrt_pd(rsq12);
261 rinv20 = gmx_mm_invsqrt_pd(rsq20);
262 rinv21 = gmx_mm_invsqrt_pd(rsq21);
263 rinv22 = gmx_mm_invsqrt_pd(rsq22);
265 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
266 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
267 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
268 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
269 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
270 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
271 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
272 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
273 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
275 fjx0 = _mm_setzero_pd();
276 fjy0 = _mm_setzero_pd();
277 fjz0 = _mm_setzero_pd();
278 fjx1 = _mm_setzero_pd();
279 fjy1 = _mm_setzero_pd();
280 fjz1 = _mm_setzero_pd();
281 fjx2 = _mm_setzero_pd();
282 fjy2 = _mm_setzero_pd();
283 fjz2 = _mm_setzero_pd();
285 /**************************
286 * CALCULATE INTERACTIONS *
287 **************************/
289 if (gmx_mm_any_lt(rsq00,rcutoff2))
292 r00 = _mm_mul_pd(rsq00,rinv00);
294 /* REACTION-FIELD ELECTROSTATICS */
295 velec = _mm_mul_pd(qq00,_mm_sub_pd(_mm_macc_pd(krf,rsq00,rinv00),crf));
296 felec = _mm_mul_pd(qq00,_mm_msub_pd(rinv00,rinvsq00,krf2));
298 /* LENNARD-JONES DISPERSION/REPULSION */
300 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
301 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
302 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
303 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
304 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
306 d = _mm_sub_pd(r00,rswitch);
307 d = _mm_max_pd(d,_mm_setzero_pd());
308 d2 = _mm_mul_pd(d,d);
309 sw = _mm_add_pd(one,_mm_mul_pd(d2,_mm_mul_pd(d,_mm_macc_pd(d,_mm_macc_pd(d,swV5,swV4),swV3))));
311 dsw = _mm_mul_pd(d2,_mm_macc_pd(d,_mm_macc_pd(d,swF4,swF3),swF2));
313 /* Evaluate switch function */
314 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
315 fvdw = _mm_msub_pd( fvdw,sw , _mm_mul_pd(rinv00,_mm_mul_pd(vvdw,dsw)) );
316 vvdw = _mm_mul_pd(vvdw,sw);
317 cutoff_mask = _mm_cmplt_pd(rsq00,rcutoff2);
319 /* Update potential sum for this i atom from the interaction with this j atom. */
320 velec = _mm_and_pd(velec,cutoff_mask);
321 velecsum = _mm_add_pd(velecsum,velec);
322 vvdw = _mm_and_pd(vvdw,cutoff_mask);
323 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
325 fscal = _mm_add_pd(felec,fvdw);
327 fscal = _mm_and_pd(fscal,cutoff_mask);
329 /* Update vectorial force */
330 fix0 = _mm_macc_pd(dx00,fscal,fix0);
331 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
332 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
334 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
335 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
336 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
340 /**************************
341 * CALCULATE INTERACTIONS *
342 **************************/
344 if (gmx_mm_any_lt(rsq01,rcutoff2))
347 /* REACTION-FIELD ELECTROSTATICS */
348 velec = _mm_mul_pd(qq01,_mm_sub_pd(_mm_macc_pd(krf,rsq01,rinv01),crf));
349 felec = _mm_mul_pd(qq01,_mm_msub_pd(rinv01,rinvsq01,krf2));
351 cutoff_mask = _mm_cmplt_pd(rsq01,rcutoff2);
353 /* Update potential sum for this i atom from the interaction with this j atom. */
354 velec = _mm_and_pd(velec,cutoff_mask);
355 velecsum = _mm_add_pd(velecsum,velec);
359 fscal = _mm_and_pd(fscal,cutoff_mask);
361 /* Update vectorial force */
362 fix0 = _mm_macc_pd(dx01,fscal,fix0);
363 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
364 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
366 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
367 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
368 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
372 /**************************
373 * CALCULATE INTERACTIONS *
374 **************************/
376 if (gmx_mm_any_lt(rsq02,rcutoff2))
379 /* REACTION-FIELD ELECTROSTATICS */
380 velec = _mm_mul_pd(qq02,_mm_sub_pd(_mm_macc_pd(krf,rsq02,rinv02),crf));
381 felec = _mm_mul_pd(qq02,_mm_msub_pd(rinv02,rinvsq02,krf2));
383 cutoff_mask = _mm_cmplt_pd(rsq02,rcutoff2);
385 /* Update potential sum for this i atom from the interaction with this j atom. */
386 velec = _mm_and_pd(velec,cutoff_mask);
387 velecsum = _mm_add_pd(velecsum,velec);
391 fscal = _mm_and_pd(fscal,cutoff_mask);
393 /* Update vectorial force */
394 fix0 = _mm_macc_pd(dx02,fscal,fix0);
395 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
396 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
398 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
399 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
400 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
404 /**************************
405 * CALCULATE INTERACTIONS *
406 **************************/
408 if (gmx_mm_any_lt(rsq10,rcutoff2))
411 /* REACTION-FIELD ELECTROSTATICS */
412 velec = _mm_mul_pd(qq10,_mm_sub_pd(_mm_macc_pd(krf,rsq10,rinv10),crf));
413 felec = _mm_mul_pd(qq10,_mm_msub_pd(rinv10,rinvsq10,krf2));
415 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
417 /* Update potential sum for this i atom from the interaction with this j atom. */
418 velec = _mm_and_pd(velec,cutoff_mask);
419 velecsum = _mm_add_pd(velecsum,velec);
423 fscal = _mm_and_pd(fscal,cutoff_mask);
425 /* Update vectorial force */
426 fix1 = _mm_macc_pd(dx10,fscal,fix1);
427 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
428 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
430 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
431 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
432 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
436 /**************************
437 * CALCULATE INTERACTIONS *
438 **************************/
440 if (gmx_mm_any_lt(rsq11,rcutoff2))
443 /* REACTION-FIELD ELECTROSTATICS */
444 velec = _mm_mul_pd(qq11,_mm_sub_pd(_mm_macc_pd(krf,rsq11,rinv11),crf));
445 felec = _mm_mul_pd(qq11,_mm_msub_pd(rinv11,rinvsq11,krf2));
447 cutoff_mask = _mm_cmplt_pd(rsq11,rcutoff2);
449 /* Update potential sum for this i atom from the interaction with this j atom. */
450 velec = _mm_and_pd(velec,cutoff_mask);
451 velecsum = _mm_add_pd(velecsum,velec);
455 fscal = _mm_and_pd(fscal,cutoff_mask);
457 /* Update vectorial force */
458 fix1 = _mm_macc_pd(dx11,fscal,fix1);
459 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
460 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
462 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
463 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
464 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
468 /**************************
469 * CALCULATE INTERACTIONS *
470 **************************/
472 if (gmx_mm_any_lt(rsq12,rcutoff2))
475 /* REACTION-FIELD ELECTROSTATICS */
476 velec = _mm_mul_pd(qq12,_mm_sub_pd(_mm_macc_pd(krf,rsq12,rinv12),crf));
477 felec = _mm_mul_pd(qq12,_mm_msub_pd(rinv12,rinvsq12,krf2));
479 cutoff_mask = _mm_cmplt_pd(rsq12,rcutoff2);
481 /* Update potential sum for this i atom from the interaction with this j atom. */
482 velec = _mm_and_pd(velec,cutoff_mask);
483 velecsum = _mm_add_pd(velecsum,velec);
487 fscal = _mm_and_pd(fscal,cutoff_mask);
489 /* Update vectorial force */
490 fix1 = _mm_macc_pd(dx12,fscal,fix1);
491 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
492 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
494 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
495 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
496 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
500 /**************************
501 * CALCULATE INTERACTIONS *
502 **************************/
504 if (gmx_mm_any_lt(rsq20,rcutoff2))
507 /* REACTION-FIELD ELECTROSTATICS */
508 velec = _mm_mul_pd(qq20,_mm_sub_pd(_mm_macc_pd(krf,rsq20,rinv20),crf));
509 felec = _mm_mul_pd(qq20,_mm_msub_pd(rinv20,rinvsq20,krf2));
511 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
513 /* Update potential sum for this i atom from the interaction with this j atom. */
514 velec = _mm_and_pd(velec,cutoff_mask);
515 velecsum = _mm_add_pd(velecsum,velec);
519 fscal = _mm_and_pd(fscal,cutoff_mask);
521 /* Update vectorial force */
522 fix2 = _mm_macc_pd(dx20,fscal,fix2);
523 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
524 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
526 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
527 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
528 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
532 /**************************
533 * CALCULATE INTERACTIONS *
534 **************************/
536 if (gmx_mm_any_lt(rsq21,rcutoff2))
539 /* REACTION-FIELD ELECTROSTATICS */
540 velec = _mm_mul_pd(qq21,_mm_sub_pd(_mm_macc_pd(krf,rsq21,rinv21),crf));
541 felec = _mm_mul_pd(qq21,_mm_msub_pd(rinv21,rinvsq21,krf2));
543 cutoff_mask = _mm_cmplt_pd(rsq21,rcutoff2);
545 /* Update potential sum for this i atom from the interaction with this j atom. */
546 velec = _mm_and_pd(velec,cutoff_mask);
547 velecsum = _mm_add_pd(velecsum,velec);
551 fscal = _mm_and_pd(fscal,cutoff_mask);
553 /* Update vectorial force */
554 fix2 = _mm_macc_pd(dx21,fscal,fix2);
555 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
556 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
558 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
559 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
560 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
564 /**************************
565 * CALCULATE INTERACTIONS *
566 **************************/
568 if (gmx_mm_any_lt(rsq22,rcutoff2))
571 /* REACTION-FIELD ELECTROSTATICS */
572 velec = _mm_mul_pd(qq22,_mm_sub_pd(_mm_macc_pd(krf,rsq22,rinv22),crf));
573 felec = _mm_mul_pd(qq22,_mm_msub_pd(rinv22,rinvsq22,krf2));
575 cutoff_mask = _mm_cmplt_pd(rsq22,rcutoff2);
577 /* Update potential sum for this i atom from the interaction with this j atom. */
578 velec = _mm_and_pd(velec,cutoff_mask);
579 velecsum = _mm_add_pd(velecsum,velec);
583 fscal = _mm_and_pd(fscal,cutoff_mask);
585 /* Update vectorial force */
586 fix2 = _mm_macc_pd(dx22,fscal,fix2);
587 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
588 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
590 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
591 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
592 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
596 gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
598 /* Inner loop uses 385 flops */
605 j_coord_offsetA = DIM*jnrA;
607 /* load j atom coordinates */
608 gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
609 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
611 /* Calculate displacement vector */
612 dx00 = _mm_sub_pd(ix0,jx0);
613 dy00 = _mm_sub_pd(iy0,jy0);
614 dz00 = _mm_sub_pd(iz0,jz0);
615 dx01 = _mm_sub_pd(ix0,jx1);
616 dy01 = _mm_sub_pd(iy0,jy1);
617 dz01 = _mm_sub_pd(iz0,jz1);
618 dx02 = _mm_sub_pd(ix0,jx2);
619 dy02 = _mm_sub_pd(iy0,jy2);
620 dz02 = _mm_sub_pd(iz0,jz2);
621 dx10 = _mm_sub_pd(ix1,jx0);
622 dy10 = _mm_sub_pd(iy1,jy0);
623 dz10 = _mm_sub_pd(iz1,jz0);
624 dx11 = _mm_sub_pd(ix1,jx1);
625 dy11 = _mm_sub_pd(iy1,jy1);
626 dz11 = _mm_sub_pd(iz1,jz1);
627 dx12 = _mm_sub_pd(ix1,jx2);
628 dy12 = _mm_sub_pd(iy1,jy2);
629 dz12 = _mm_sub_pd(iz1,jz2);
630 dx20 = _mm_sub_pd(ix2,jx0);
631 dy20 = _mm_sub_pd(iy2,jy0);
632 dz20 = _mm_sub_pd(iz2,jz0);
633 dx21 = _mm_sub_pd(ix2,jx1);
634 dy21 = _mm_sub_pd(iy2,jy1);
635 dz21 = _mm_sub_pd(iz2,jz1);
636 dx22 = _mm_sub_pd(ix2,jx2);
637 dy22 = _mm_sub_pd(iy2,jy2);
638 dz22 = _mm_sub_pd(iz2,jz2);
640 /* Calculate squared distance and things based on it */
641 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
642 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
643 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
644 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
645 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
646 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
647 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
648 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
649 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
651 rinv00 = gmx_mm_invsqrt_pd(rsq00);
652 rinv01 = gmx_mm_invsqrt_pd(rsq01);
653 rinv02 = gmx_mm_invsqrt_pd(rsq02);
654 rinv10 = gmx_mm_invsqrt_pd(rsq10);
655 rinv11 = gmx_mm_invsqrt_pd(rsq11);
656 rinv12 = gmx_mm_invsqrt_pd(rsq12);
657 rinv20 = gmx_mm_invsqrt_pd(rsq20);
658 rinv21 = gmx_mm_invsqrt_pd(rsq21);
659 rinv22 = gmx_mm_invsqrt_pd(rsq22);
661 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
662 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
663 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
664 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
665 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
666 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
667 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
668 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
669 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
671 fjx0 = _mm_setzero_pd();
672 fjy0 = _mm_setzero_pd();
673 fjz0 = _mm_setzero_pd();
674 fjx1 = _mm_setzero_pd();
675 fjy1 = _mm_setzero_pd();
676 fjz1 = _mm_setzero_pd();
677 fjx2 = _mm_setzero_pd();
678 fjy2 = _mm_setzero_pd();
679 fjz2 = _mm_setzero_pd();
681 /**************************
682 * CALCULATE INTERACTIONS *
683 **************************/
685 if (gmx_mm_any_lt(rsq00,rcutoff2))
688 r00 = _mm_mul_pd(rsq00,rinv00);
690 /* REACTION-FIELD ELECTROSTATICS */
691 velec = _mm_mul_pd(qq00,_mm_sub_pd(_mm_macc_pd(krf,rsq00,rinv00),crf));
692 felec = _mm_mul_pd(qq00,_mm_msub_pd(rinv00,rinvsq00,krf2));
694 /* LENNARD-JONES DISPERSION/REPULSION */
696 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
697 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
698 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
699 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
700 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
702 d = _mm_sub_pd(r00,rswitch);
703 d = _mm_max_pd(d,_mm_setzero_pd());
704 d2 = _mm_mul_pd(d,d);
705 sw = _mm_add_pd(one,_mm_mul_pd(d2,_mm_mul_pd(d,_mm_macc_pd(d,_mm_macc_pd(d,swV5,swV4),swV3))));
707 dsw = _mm_mul_pd(d2,_mm_macc_pd(d,_mm_macc_pd(d,swF4,swF3),swF2));
709 /* Evaluate switch function */
710 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
711 fvdw = _mm_msub_pd( fvdw,sw , _mm_mul_pd(rinv00,_mm_mul_pd(vvdw,dsw)) );
712 vvdw = _mm_mul_pd(vvdw,sw);
713 cutoff_mask = _mm_cmplt_pd(rsq00,rcutoff2);
715 /* Update potential sum for this i atom from the interaction with this j atom. */
716 velec = _mm_and_pd(velec,cutoff_mask);
717 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
718 velecsum = _mm_add_pd(velecsum,velec);
719 vvdw = _mm_and_pd(vvdw,cutoff_mask);
720 vvdw = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
721 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
723 fscal = _mm_add_pd(felec,fvdw);
725 fscal = _mm_and_pd(fscal,cutoff_mask);
727 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
729 /* Update vectorial force */
730 fix0 = _mm_macc_pd(dx00,fscal,fix0);
731 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
732 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
734 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
735 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
736 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
740 /**************************
741 * CALCULATE INTERACTIONS *
742 **************************/
744 if (gmx_mm_any_lt(rsq01,rcutoff2))
747 /* REACTION-FIELD ELECTROSTATICS */
748 velec = _mm_mul_pd(qq01,_mm_sub_pd(_mm_macc_pd(krf,rsq01,rinv01),crf));
749 felec = _mm_mul_pd(qq01,_mm_msub_pd(rinv01,rinvsq01,krf2));
751 cutoff_mask = _mm_cmplt_pd(rsq01,rcutoff2);
753 /* Update potential sum for this i atom from the interaction with this j atom. */
754 velec = _mm_and_pd(velec,cutoff_mask);
755 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
756 velecsum = _mm_add_pd(velecsum,velec);
760 fscal = _mm_and_pd(fscal,cutoff_mask);
762 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
764 /* Update vectorial force */
765 fix0 = _mm_macc_pd(dx01,fscal,fix0);
766 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
767 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
769 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
770 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
771 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
775 /**************************
776 * CALCULATE INTERACTIONS *
777 **************************/
779 if (gmx_mm_any_lt(rsq02,rcutoff2))
782 /* REACTION-FIELD ELECTROSTATICS */
783 velec = _mm_mul_pd(qq02,_mm_sub_pd(_mm_macc_pd(krf,rsq02,rinv02),crf));
784 felec = _mm_mul_pd(qq02,_mm_msub_pd(rinv02,rinvsq02,krf2));
786 cutoff_mask = _mm_cmplt_pd(rsq02,rcutoff2);
788 /* Update potential sum for this i atom from the interaction with this j atom. */
789 velec = _mm_and_pd(velec,cutoff_mask);
790 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
791 velecsum = _mm_add_pd(velecsum,velec);
795 fscal = _mm_and_pd(fscal,cutoff_mask);
797 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
799 /* Update vectorial force */
800 fix0 = _mm_macc_pd(dx02,fscal,fix0);
801 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
802 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
804 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
805 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
806 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
810 /**************************
811 * CALCULATE INTERACTIONS *
812 **************************/
814 if (gmx_mm_any_lt(rsq10,rcutoff2))
817 /* REACTION-FIELD ELECTROSTATICS */
818 velec = _mm_mul_pd(qq10,_mm_sub_pd(_mm_macc_pd(krf,rsq10,rinv10),crf));
819 felec = _mm_mul_pd(qq10,_mm_msub_pd(rinv10,rinvsq10,krf2));
821 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
823 /* Update potential sum for this i atom from the interaction with this j atom. */
824 velec = _mm_and_pd(velec,cutoff_mask);
825 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
826 velecsum = _mm_add_pd(velecsum,velec);
830 fscal = _mm_and_pd(fscal,cutoff_mask);
832 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
834 /* Update vectorial force */
835 fix1 = _mm_macc_pd(dx10,fscal,fix1);
836 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
837 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
839 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
840 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
841 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
845 /**************************
846 * CALCULATE INTERACTIONS *
847 **************************/
849 if (gmx_mm_any_lt(rsq11,rcutoff2))
852 /* REACTION-FIELD ELECTROSTATICS */
853 velec = _mm_mul_pd(qq11,_mm_sub_pd(_mm_macc_pd(krf,rsq11,rinv11),crf));
854 felec = _mm_mul_pd(qq11,_mm_msub_pd(rinv11,rinvsq11,krf2));
856 cutoff_mask = _mm_cmplt_pd(rsq11,rcutoff2);
858 /* Update potential sum for this i atom from the interaction with this j atom. */
859 velec = _mm_and_pd(velec,cutoff_mask);
860 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
861 velecsum = _mm_add_pd(velecsum,velec);
865 fscal = _mm_and_pd(fscal,cutoff_mask);
867 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
869 /* Update vectorial force */
870 fix1 = _mm_macc_pd(dx11,fscal,fix1);
871 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
872 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
874 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
875 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
876 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
880 /**************************
881 * CALCULATE INTERACTIONS *
882 **************************/
884 if (gmx_mm_any_lt(rsq12,rcutoff2))
887 /* REACTION-FIELD ELECTROSTATICS */
888 velec = _mm_mul_pd(qq12,_mm_sub_pd(_mm_macc_pd(krf,rsq12,rinv12),crf));
889 felec = _mm_mul_pd(qq12,_mm_msub_pd(rinv12,rinvsq12,krf2));
891 cutoff_mask = _mm_cmplt_pd(rsq12,rcutoff2);
893 /* Update potential sum for this i atom from the interaction with this j atom. */
894 velec = _mm_and_pd(velec,cutoff_mask);
895 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
896 velecsum = _mm_add_pd(velecsum,velec);
900 fscal = _mm_and_pd(fscal,cutoff_mask);
902 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
904 /* Update vectorial force */
905 fix1 = _mm_macc_pd(dx12,fscal,fix1);
906 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
907 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
909 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
910 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
911 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
915 /**************************
916 * CALCULATE INTERACTIONS *
917 **************************/
919 if (gmx_mm_any_lt(rsq20,rcutoff2))
922 /* REACTION-FIELD ELECTROSTATICS */
923 velec = _mm_mul_pd(qq20,_mm_sub_pd(_mm_macc_pd(krf,rsq20,rinv20),crf));
924 felec = _mm_mul_pd(qq20,_mm_msub_pd(rinv20,rinvsq20,krf2));
926 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
928 /* Update potential sum for this i atom from the interaction with this j atom. */
929 velec = _mm_and_pd(velec,cutoff_mask);
930 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
931 velecsum = _mm_add_pd(velecsum,velec);
935 fscal = _mm_and_pd(fscal,cutoff_mask);
937 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
939 /* Update vectorial force */
940 fix2 = _mm_macc_pd(dx20,fscal,fix2);
941 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
942 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
944 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
945 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
946 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
950 /**************************
951 * CALCULATE INTERACTIONS *
952 **************************/
954 if (gmx_mm_any_lt(rsq21,rcutoff2))
957 /* REACTION-FIELD ELECTROSTATICS */
958 velec = _mm_mul_pd(qq21,_mm_sub_pd(_mm_macc_pd(krf,rsq21,rinv21),crf));
959 felec = _mm_mul_pd(qq21,_mm_msub_pd(rinv21,rinvsq21,krf2));
961 cutoff_mask = _mm_cmplt_pd(rsq21,rcutoff2);
963 /* Update potential sum for this i atom from the interaction with this j atom. */
964 velec = _mm_and_pd(velec,cutoff_mask);
965 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
966 velecsum = _mm_add_pd(velecsum,velec);
970 fscal = _mm_and_pd(fscal,cutoff_mask);
972 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
974 /* Update vectorial force */
975 fix2 = _mm_macc_pd(dx21,fscal,fix2);
976 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
977 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
979 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
980 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
981 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
985 /**************************
986 * CALCULATE INTERACTIONS *
987 **************************/
989 if (gmx_mm_any_lt(rsq22,rcutoff2))
992 /* REACTION-FIELD ELECTROSTATICS */
993 velec = _mm_mul_pd(qq22,_mm_sub_pd(_mm_macc_pd(krf,rsq22,rinv22),crf));
994 felec = _mm_mul_pd(qq22,_mm_msub_pd(rinv22,rinvsq22,krf2));
996 cutoff_mask = _mm_cmplt_pd(rsq22,rcutoff2);
998 /* Update potential sum for this i atom from the interaction with this j atom. */
999 velec = _mm_and_pd(velec,cutoff_mask);
1000 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
1001 velecsum = _mm_add_pd(velecsum,velec);
1005 fscal = _mm_and_pd(fscal,cutoff_mask);
1007 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1009 /* Update vectorial force */
1010 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1011 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1012 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1014 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1015 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1016 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1020 gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1022 /* Inner loop uses 385 flops */
1025 /* End of innermost loop */
1027 gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1028 f+i_coord_offset,fshift+i_shift_offset);
1031 /* Update potential energies */
1032 gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
1033 gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
1035 /* Increment number of inner iterations */
1036 inneriter += j_index_end - j_index_start;
1038 /* Outer loop uses 20 flops */
1041 /* Increment number of outer iterations */
1044 /* Update outer/inner flops */
1046 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*385);
1049 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwLJSw_GeomW3W3_F_avx_128_fma_double
1050 * Electrostatics interaction: ReactionField
1051 * VdW interaction: LennardJones
1052 * Geometry: Water3-Water3
1053 * Calculate force/pot: Force
1056 nb_kernel_ElecRFCut_VdwLJSw_GeomW3W3_F_avx_128_fma_double
1057 (t_nblist * gmx_restrict nlist,
1058 rvec * gmx_restrict xx,
1059 rvec * gmx_restrict ff,
1060 t_forcerec * gmx_restrict fr,
1061 t_mdatoms * gmx_restrict mdatoms,
1062 nb_kernel_data_t * gmx_restrict kernel_data,
1063 t_nrnb * gmx_restrict nrnb)
1065 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1066 * just 0 for non-waters.
1067 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
1068 * jnr indices corresponding to data put in the four positions in the SIMD register.
1070 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1071 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1073 int j_coord_offsetA,j_coord_offsetB;
1074 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1075 real rcutoff_scalar;
1076 real *shiftvec,*fshift,*x,*f;
1077 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
1079 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1081 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1083 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1084 int vdwjidx0A,vdwjidx0B;
1085 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1086 int vdwjidx1A,vdwjidx1B;
1087 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1088 int vdwjidx2A,vdwjidx2B;
1089 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1090 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1091 __m128d dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1092 __m128d dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1093 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1094 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1095 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1096 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1097 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1098 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1099 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
1102 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1105 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
1106 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
1107 __m128d rswitch,swV3,swV4,swV5,swF2,swF3,swF4,d,d2,sw,dsw;
1108 real rswitch_scalar,d_scalar;
1109 __m128d dummy_mask,cutoff_mask;
1110 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
1111 __m128d one = _mm_set1_pd(1.0);
1112 __m128d two = _mm_set1_pd(2.0);
1118 jindex = nlist->jindex;
1120 shiftidx = nlist->shift;
1122 shiftvec = fr->shift_vec[0];
1123 fshift = fr->fshift[0];
1124 facel = _mm_set1_pd(fr->epsfac);
1125 charge = mdatoms->chargeA;
1126 krf = _mm_set1_pd(fr->ic->k_rf);
1127 krf2 = _mm_set1_pd(fr->ic->k_rf*2.0);
1128 crf = _mm_set1_pd(fr->ic->c_rf);
1129 nvdwtype = fr->ntype;
1130 vdwparam = fr->nbfp;
1131 vdwtype = mdatoms->typeA;
1133 /* Setup water-specific parameters */
1134 inr = nlist->iinr[0];
1135 iq0 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
1136 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
1137 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
1138 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
1140 jq0 = _mm_set1_pd(charge[inr+0]);
1141 jq1 = _mm_set1_pd(charge[inr+1]);
1142 jq2 = _mm_set1_pd(charge[inr+2]);
1143 vdwjidx0A = 2*vdwtype[inr+0];
1144 qq00 = _mm_mul_pd(iq0,jq0);
1145 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
1146 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
1147 qq01 = _mm_mul_pd(iq0,jq1);
1148 qq02 = _mm_mul_pd(iq0,jq2);
1149 qq10 = _mm_mul_pd(iq1,jq0);
1150 qq11 = _mm_mul_pd(iq1,jq1);
1151 qq12 = _mm_mul_pd(iq1,jq2);
1152 qq20 = _mm_mul_pd(iq2,jq0);
1153 qq21 = _mm_mul_pd(iq2,jq1);
1154 qq22 = _mm_mul_pd(iq2,jq2);
1156 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
1157 rcutoff_scalar = fr->rcoulomb;
1158 rcutoff = _mm_set1_pd(rcutoff_scalar);
1159 rcutoff2 = _mm_mul_pd(rcutoff,rcutoff);
1161 rswitch_scalar = fr->rvdw_switch;
1162 rswitch = _mm_set1_pd(rswitch_scalar);
1163 /* Setup switch parameters */
1164 d_scalar = rcutoff_scalar-rswitch_scalar;
1165 d = _mm_set1_pd(d_scalar);
1166 swV3 = _mm_set1_pd(-10.0/(d_scalar*d_scalar*d_scalar));
1167 swV4 = _mm_set1_pd( 15.0/(d_scalar*d_scalar*d_scalar*d_scalar));
1168 swV5 = _mm_set1_pd( -6.0/(d_scalar*d_scalar*d_scalar*d_scalar*d_scalar));
1169 swF2 = _mm_set1_pd(-30.0/(d_scalar*d_scalar*d_scalar));
1170 swF3 = _mm_set1_pd( 60.0/(d_scalar*d_scalar*d_scalar*d_scalar));
1171 swF4 = _mm_set1_pd(-30.0/(d_scalar*d_scalar*d_scalar*d_scalar*d_scalar));
1173 /* Avoid stupid compiler warnings */
1175 j_coord_offsetA = 0;
1176 j_coord_offsetB = 0;
1181 /* Start outer loop over neighborlists */
1182 for(iidx=0; iidx<nri; iidx++)
1184 /* Load shift vector for this list */
1185 i_shift_offset = DIM*shiftidx[iidx];
1187 /* Load limits for loop over neighbors */
1188 j_index_start = jindex[iidx];
1189 j_index_end = jindex[iidx+1];
1191 /* Get outer coordinate index */
1193 i_coord_offset = DIM*inr;
1195 /* Load i particle coords and add shift vector */
1196 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
1197 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1199 fix0 = _mm_setzero_pd();
1200 fiy0 = _mm_setzero_pd();
1201 fiz0 = _mm_setzero_pd();
1202 fix1 = _mm_setzero_pd();
1203 fiy1 = _mm_setzero_pd();
1204 fiz1 = _mm_setzero_pd();
1205 fix2 = _mm_setzero_pd();
1206 fiy2 = _mm_setzero_pd();
1207 fiz2 = _mm_setzero_pd();
1209 /* Start inner kernel loop */
1210 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
1213 /* Get j neighbor index, and coordinate index */
1215 jnrB = jjnr[jidx+1];
1216 j_coord_offsetA = DIM*jnrA;
1217 j_coord_offsetB = DIM*jnrB;
1219 /* load j atom coordinates */
1220 gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1221 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1223 /* Calculate displacement vector */
1224 dx00 = _mm_sub_pd(ix0,jx0);
1225 dy00 = _mm_sub_pd(iy0,jy0);
1226 dz00 = _mm_sub_pd(iz0,jz0);
1227 dx01 = _mm_sub_pd(ix0,jx1);
1228 dy01 = _mm_sub_pd(iy0,jy1);
1229 dz01 = _mm_sub_pd(iz0,jz1);
1230 dx02 = _mm_sub_pd(ix0,jx2);
1231 dy02 = _mm_sub_pd(iy0,jy2);
1232 dz02 = _mm_sub_pd(iz0,jz2);
1233 dx10 = _mm_sub_pd(ix1,jx0);
1234 dy10 = _mm_sub_pd(iy1,jy0);
1235 dz10 = _mm_sub_pd(iz1,jz0);
1236 dx11 = _mm_sub_pd(ix1,jx1);
1237 dy11 = _mm_sub_pd(iy1,jy1);
1238 dz11 = _mm_sub_pd(iz1,jz1);
1239 dx12 = _mm_sub_pd(ix1,jx2);
1240 dy12 = _mm_sub_pd(iy1,jy2);
1241 dz12 = _mm_sub_pd(iz1,jz2);
1242 dx20 = _mm_sub_pd(ix2,jx0);
1243 dy20 = _mm_sub_pd(iy2,jy0);
1244 dz20 = _mm_sub_pd(iz2,jz0);
1245 dx21 = _mm_sub_pd(ix2,jx1);
1246 dy21 = _mm_sub_pd(iy2,jy1);
1247 dz21 = _mm_sub_pd(iz2,jz1);
1248 dx22 = _mm_sub_pd(ix2,jx2);
1249 dy22 = _mm_sub_pd(iy2,jy2);
1250 dz22 = _mm_sub_pd(iz2,jz2);
1252 /* Calculate squared distance and things based on it */
1253 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1254 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1255 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1256 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1257 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1258 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1259 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1260 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1261 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1263 rinv00 = gmx_mm_invsqrt_pd(rsq00);
1264 rinv01 = gmx_mm_invsqrt_pd(rsq01);
1265 rinv02 = gmx_mm_invsqrt_pd(rsq02);
1266 rinv10 = gmx_mm_invsqrt_pd(rsq10);
1267 rinv11 = gmx_mm_invsqrt_pd(rsq11);
1268 rinv12 = gmx_mm_invsqrt_pd(rsq12);
1269 rinv20 = gmx_mm_invsqrt_pd(rsq20);
1270 rinv21 = gmx_mm_invsqrt_pd(rsq21);
1271 rinv22 = gmx_mm_invsqrt_pd(rsq22);
1273 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
1274 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
1275 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
1276 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
1277 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
1278 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
1279 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
1280 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
1281 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
1283 fjx0 = _mm_setzero_pd();
1284 fjy0 = _mm_setzero_pd();
1285 fjz0 = _mm_setzero_pd();
1286 fjx1 = _mm_setzero_pd();
1287 fjy1 = _mm_setzero_pd();
1288 fjz1 = _mm_setzero_pd();
1289 fjx2 = _mm_setzero_pd();
1290 fjy2 = _mm_setzero_pd();
1291 fjz2 = _mm_setzero_pd();
1293 /**************************
1294 * CALCULATE INTERACTIONS *
1295 **************************/
1297 if (gmx_mm_any_lt(rsq00,rcutoff2))
1300 r00 = _mm_mul_pd(rsq00,rinv00);
1302 /* REACTION-FIELD ELECTROSTATICS */
1303 felec = _mm_mul_pd(qq00,_mm_msub_pd(rinv00,rinvsq00,krf2));
1305 /* LENNARD-JONES DISPERSION/REPULSION */
1307 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1308 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
1309 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
1310 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
1311 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
1313 d = _mm_sub_pd(r00,rswitch);
1314 d = _mm_max_pd(d,_mm_setzero_pd());
1315 d2 = _mm_mul_pd(d,d);
1316 sw = _mm_add_pd(one,_mm_mul_pd(d2,_mm_mul_pd(d,_mm_macc_pd(d,_mm_macc_pd(d,swV5,swV4),swV3))));
1318 dsw = _mm_mul_pd(d2,_mm_macc_pd(d,_mm_macc_pd(d,swF4,swF3),swF2));
1320 /* Evaluate switch function */
1321 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1322 fvdw = _mm_msub_pd( fvdw,sw , _mm_mul_pd(rinv00,_mm_mul_pd(vvdw,dsw)) );
1323 cutoff_mask = _mm_cmplt_pd(rsq00,rcutoff2);
1325 fscal = _mm_add_pd(felec,fvdw);
1327 fscal = _mm_and_pd(fscal,cutoff_mask);
1329 /* Update vectorial force */
1330 fix0 = _mm_macc_pd(dx00,fscal,fix0);
1331 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
1332 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
1334 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
1335 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
1336 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
1340 /**************************
1341 * CALCULATE INTERACTIONS *
1342 **************************/
1344 if (gmx_mm_any_lt(rsq01,rcutoff2))
1347 /* REACTION-FIELD ELECTROSTATICS */
1348 felec = _mm_mul_pd(qq01,_mm_msub_pd(rinv01,rinvsq01,krf2));
1350 cutoff_mask = _mm_cmplt_pd(rsq01,rcutoff2);
1354 fscal = _mm_and_pd(fscal,cutoff_mask);
1356 /* Update vectorial force */
1357 fix0 = _mm_macc_pd(dx01,fscal,fix0);
1358 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
1359 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
1361 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
1362 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
1363 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
1367 /**************************
1368 * CALCULATE INTERACTIONS *
1369 **************************/
1371 if (gmx_mm_any_lt(rsq02,rcutoff2))
1374 /* REACTION-FIELD ELECTROSTATICS */
1375 felec = _mm_mul_pd(qq02,_mm_msub_pd(rinv02,rinvsq02,krf2));
1377 cutoff_mask = _mm_cmplt_pd(rsq02,rcutoff2);
1381 fscal = _mm_and_pd(fscal,cutoff_mask);
1383 /* Update vectorial force */
1384 fix0 = _mm_macc_pd(dx02,fscal,fix0);
1385 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
1386 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
1388 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
1389 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
1390 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
1394 /**************************
1395 * CALCULATE INTERACTIONS *
1396 **************************/
1398 if (gmx_mm_any_lt(rsq10,rcutoff2))
1401 /* REACTION-FIELD ELECTROSTATICS */
1402 felec = _mm_mul_pd(qq10,_mm_msub_pd(rinv10,rinvsq10,krf2));
1404 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
1408 fscal = _mm_and_pd(fscal,cutoff_mask);
1410 /* Update vectorial force */
1411 fix1 = _mm_macc_pd(dx10,fscal,fix1);
1412 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
1413 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
1415 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
1416 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
1417 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
1421 /**************************
1422 * CALCULATE INTERACTIONS *
1423 **************************/
1425 if (gmx_mm_any_lt(rsq11,rcutoff2))
1428 /* REACTION-FIELD ELECTROSTATICS */
1429 felec = _mm_mul_pd(qq11,_mm_msub_pd(rinv11,rinvsq11,krf2));
1431 cutoff_mask = _mm_cmplt_pd(rsq11,rcutoff2);
1435 fscal = _mm_and_pd(fscal,cutoff_mask);
1437 /* Update vectorial force */
1438 fix1 = _mm_macc_pd(dx11,fscal,fix1);
1439 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
1440 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
1442 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
1443 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
1444 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
1448 /**************************
1449 * CALCULATE INTERACTIONS *
1450 **************************/
1452 if (gmx_mm_any_lt(rsq12,rcutoff2))
1455 /* REACTION-FIELD ELECTROSTATICS */
1456 felec = _mm_mul_pd(qq12,_mm_msub_pd(rinv12,rinvsq12,krf2));
1458 cutoff_mask = _mm_cmplt_pd(rsq12,rcutoff2);
1462 fscal = _mm_and_pd(fscal,cutoff_mask);
1464 /* Update vectorial force */
1465 fix1 = _mm_macc_pd(dx12,fscal,fix1);
1466 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
1467 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1469 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1470 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1471 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1475 /**************************
1476 * CALCULATE INTERACTIONS *
1477 **************************/
1479 if (gmx_mm_any_lt(rsq20,rcutoff2))
1482 /* REACTION-FIELD ELECTROSTATICS */
1483 felec = _mm_mul_pd(qq20,_mm_msub_pd(rinv20,rinvsq20,krf2));
1485 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
1489 fscal = _mm_and_pd(fscal,cutoff_mask);
1491 /* Update vectorial force */
1492 fix2 = _mm_macc_pd(dx20,fscal,fix2);
1493 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
1494 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
1496 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
1497 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
1498 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
1502 /**************************
1503 * CALCULATE INTERACTIONS *
1504 **************************/
1506 if (gmx_mm_any_lt(rsq21,rcutoff2))
1509 /* REACTION-FIELD ELECTROSTATICS */
1510 felec = _mm_mul_pd(qq21,_mm_msub_pd(rinv21,rinvsq21,krf2));
1512 cutoff_mask = _mm_cmplt_pd(rsq21,rcutoff2);
1516 fscal = _mm_and_pd(fscal,cutoff_mask);
1518 /* Update vectorial force */
1519 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1520 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1521 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1523 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1524 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1525 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1529 /**************************
1530 * CALCULATE INTERACTIONS *
1531 **************************/
1533 if (gmx_mm_any_lt(rsq22,rcutoff2))
1536 /* REACTION-FIELD ELECTROSTATICS */
1537 felec = _mm_mul_pd(qq22,_mm_msub_pd(rinv22,rinvsq22,krf2));
1539 cutoff_mask = _mm_cmplt_pd(rsq22,rcutoff2);
1543 fscal = _mm_and_pd(fscal,cutoff_mask);
1545 /* Update vectorial force */
1546 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1547 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1548 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1550 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1551 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1552 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1556 gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1558 /* Inner loop uses 328 flops */
1561 if(jidx<j_index_end)
1565 j_coord_offsetA = DIM*jnrA;
1567 /* load j atom coordinates */
1568 gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
1569 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1571 /* Calculate displacement vector */
1572 dx00 = _mm_sub_pd(ix0,jx0);
1573 dy00 = _mm_sub_pd(iy0,jy0);
1574 dz00 = _mm_sub_pd(iz0,jz0);
1575 dx01 = _mm_sub_pd(ix0,jx1);
1576 dy01 = _mm_sub_pd(iy0,jy1);
1577 dz01 = _mm_sub_pd(iz0,jz1);
1578 dx02 = _mm_sub_pd(ix0,jx2);
1579 dy02 = _mm_sub_pd(iy0,jy2);
1580 dz02 = _mm_sub_pd(iz0,jz2);
1581 dx10 = _mm_sub_pd(ix1,jx0);
1582 dy10 = _mm_sub_pd(iy1,jy0);
1583 dz10 = _mm_sub_pd(iz1,jz0);
1584 dx11 = _mm_sub_pd(ix1,jx1);
1585 dy11 = _mm_sub_pd(iy1,jy1);
1586 dz11 = _mm_sub_pd(iz1,jz1);
1587 dx12 = _mm_sub_pd(ix1,jx2);
1588 dy12 = _mm_sub_pd(iy1,jy2);
1589 dz12 = _mm_sub_pd(iz1,jz2);
1590 dx20 = _mm_sub_pd(ix2,jx0);
1591 dy20 = _mm_sub_pd(iy2,jy0);
1592 dz20 = _mm_sub_pd(iz2,jz0);
1593 dx21 = _mm_sub_pd(ix2,jx1);
1594 dy21 = _mm_sub_pd(iy2,jy1);
1595 dz21 = _mm_sub_pd(iz2,jz1);
1596 dx22 = _mm_sub_pd(ix2,jx2);
1597 dy22 = _mm_sub_pd(iy2,jy2);
1598 dz22 = _mm_sub_pd(iz2,jz2);
1600 /* Calculate squared distance and things based on it */
1601 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1602 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1603 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1604 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1605 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1606 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1607 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1608 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1609 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1611 rinv00 = gmx_mm_invsqrt_pd(rsq00);
1612 rinv01 = gmx_mm_invsqrt_pd(rsq01);
1613 rinv02 = gmx_mm_invsqrt_pd(rsq02);
1614 rinv10 = gmx_mm_invsqrt_pd(rsq10);
1615 rinv11 = gmx_mm_invsqrt_pd(rsq11);
1616 rinv12 = gmx_mm_invsqrt_pd(rsq12);
1617 rinv20 = gmx_mm_invsqrt_pd(rsq20);
1618 rinv21 = gmx_mm_invsqrt_pd(rsq21);
1619 rinv22 = gmx_mm_invsqrt_pd(rsq22);
1621 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
1622 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
1623 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
1624 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
1625 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
1626 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
1627 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
1628 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
1629 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
1631 fjx0 = _mm_setzero_pd();
1632 fjy0 = _mm_setzero_pd();
1633 fjz0 = _mm_setzero_pd();
1634 fjx1 = _mm_setzero_pd();
1635 fjy1 = _mm_setzero_pd();
1636 fjz1 = _mm_setzero_pd();
1637 fjx2 = _mm_setzero_pd();
1638 fjy2 = _mm_setzero_pd();
1639 fjz2 = _mm_setzero_pd();
1641 /**************************
1642 * CALCULATE INTERACTIONS *
1643 **************************/
1645 if (gmx_mm_any_lt(rsq00,rcutoff2))
1648 r00 = _mm_mul_pd(rsq00,rinv00);
1650 /* REACTION-FIELD ELECTROSTATICS */
1651 felec = _mm_mul_pd(qq00,_mm_msub_pd(rinv00,rinvsq00,krf2));
1653 /* LENNARD-JONES DISPERSION/REPULSION */
1655 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1656 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
1657 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
1658 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
1659 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
1661 d = _mm_sub_pd(r00,rswitch);
1662 d = _mm_max_pd(d,_mm_setzero_pd());
1663 d2 = _mm_mul_pd(d,d);
1664 sw = _mm_add_pd(one,_mm_mul_pd(d2,_mm_mul_pd(d,_mm_macc_pd(d,_mm_macc_pd(d,swV5,swV4),swV3))));
1666 dsw = _mm_mul_pd(d2,_mm_macc_pd(d,_mm_macc_pd(d,swF4,swF3),swF2));
1668 /* Evaluate switch function */
1669 /* fscal'=f'/r=-(v*sw)'/r=-(v'*sw+v*dsw)/r=-v'*sw/r-v*dsw/r=fscal*sw-v*dsw/r */
1670 fvdw = _mm_msub_pd( fvdw,sw , _mm_mul_pd(rinv00,_mm_mul_pd(vvdw,dsw)) );
1671 cutoff_mask = _mm_cmplt_pd(rsq00,rcutoff2);
1673 fscal = _mm_add_pd(felec,fvdw);
1675 fscal = _mm_and_pd(fscal,cutoff_mask);
1677 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1679 /* Update vectorial force */
1680 fix0 = _mm_macc_pd(dx00,fscal,fix0);
1681 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
1682 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
1684 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
1685 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
1686 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
1690 /**************************
1691 * CALCULATE INTERACTIONS *
1692 **************************/
1694 if (gmx_mm_any_lt(rsq01,rcutoff2))
1697 /* REACTION-FIELD ELECTROSTATICS */
1698 felec = _mm_mul_pd(qq01,_mm_msub_pd(rinv01,rinvsq01,krf2));
1700 cutoff_mask = _mm_cmplt_pd(rsq01,rcutoff2);
1704 fscal = _mm_and_pd(fscal,cutoff_mask);
1706 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1708 /* Update vectorial force */
1709 fix0 = _mm_macc_pd(dx01,fscal,fix0);
1710 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
1711 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
1713 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
1714 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
1715 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
1719 /**************************
1720 * CALCULATE INTERACTIONS *
1721 **************************/
1723 if (gmx_mm_any_lt(rsq02,rcutoff2))
1726 /* REACTION-FIELD ELECTROSTATICS */
1727 felec = _mm_mul_pd(qq02,_mm_msub_pd(rinv02,rinvsq02,krf2));
1729 cutoff_mask = _mm_cmplt_pd(rsq02,rcutoff2);
1733 fscal = _mm_and_pd(fscal,cutoff_mask);
1735 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1737 /* Update vectorial force */
1738 fix0 = _mm_macc_pd(dx02,fscal,fix0);
1739 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
1740 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
1742 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
1743 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
1744 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
1748 /**************************
1749 * CALCULATE INTERACTIONS *
1750 **************************/
1752 if (gmx_mm_any_lt(rsq10,rcutoff2))
1755 /* REACTION-FIELD ELECTROSTATICS */
1756 felec = _mm_mul_pd(qq10,_mm_msub_pd(rinv10,rinvsq10,krf2));
1758 cutoff_mask = _mm_cmplt_pd(rsq10,rcutoff2);
1762 fscal = _mm_and_pd(fscal,cutoff_mask);
1764 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1766 /* Update vectorial force */
1767 fix1 = _mm_macc_pd(dx10,fscal,fix1);
1768 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
1769 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
1771 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
1772 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
1773 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
1777 /**************************
1778 * CALCULATE INTERACTIONS *
1779 **************************/
1781 if (gmx_mm_any_lt(rsq11,rcutoff2))
1784 /* REACTION-FIELD ELECTROSTATICS */
1785 felec = _mm_mul_pd(qq11,_mm_msub_pd(rinv11,rinvsq11,krf2));
1787 cutoff_mask = _mm_cmplt_pd(rsq11,rcutoff2);
1791 fscal = _mm_and_pd(fscal,cutoff_mask);
1793 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1795 /* Update vectorial force */
1796 fix1 = _mm_macc_pd(dx11,fscal,fix1);
1797 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
1798 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
1800 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
1801 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
1802 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
1806 /**************************
1807 * CALCULATE INTERACTIONS *
1808 **************************/
1810 if (gmx_mm_any_lt(rsq12,rcutoff2))
1813 /* REACTION-FIELD ELECTROSTATICS */
1814 felec = _mm_mul_pd(qq12,_mm_msub_pd(rinv12,rinvsq12,krf2));
1816 cutoff_mask = _mm_cmplt_pd(rsq12,rcutoff2);
1820 fscal = _mm_and_pd(fscal,cutoff_mask);
1822 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1824 /* Update vectorial force */
1825 fix1 = _mm_macc_pd(dx12,fscal,fix1);
1826 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
1827 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1829 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1830 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1831 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1835 /**************************
1836 * CALCULATE INTERACTIONS *
1837 **************************/
1839 if (gmx_mm_any_lt(rsq20,rcutoff2))
1842 /* REACTION-FIELD ELECTROSTATICS */
1843 felec = _mm_mul_pd(qq20,_mm_msub_pd(rinv20,rinvsq20,krf2));
1845 cutoff_mask = _mm_cmplt_pd(rsq20,rcutoff2);
1849 fscal = _mm_and_pd(fscal,cutoff_mask);
1851 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1853 /* Update vectorial force */
1854 fix2 = _mm_macc_pd(dx20,fscal,fix2);
1855 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
1856 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
1858 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
1859 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
1860 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
1864 /**************************
1865 * CALCULATE INTERACTIONS *
1866 **************************/
1868 if (gmx_mm_any_lt(rsq21,rcutoff2))
1871 /* REACTION-FIELD ELECTROSTATICS */
1872 felec = _mm_mul_pd(qq21,_mm_msub_pd(rinv21,rinvsq21,krf2));
1874 cutoff_mask = _mm_cmplt_pd(rsq21,rcutoff2);
1878 fscal = _mm_and_pd(fscal,cutoff_mask);
1880 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1882 /* Update vectorial force */
1883 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1884 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1885 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1887 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1888 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1889 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1893 /**************************
1894 * CALCULATE INTERACTIONS *
1895 **************************/
1897 if (gmx_mm_any_lt(rsq22,rcutoff2))
1900 /* REACTION-FIELD ELECTROSTATICS */
1901 felec = _mm_mul_pd(qq22,_mm_msub_pd(rinv22,rinvsq22,krf2));
1903 cutoff_mask = _mm_cmplt_pd(rsq22,rcutoff2);
1907 fscal = _mm_and_pd(fscal,cutoff_mask);
1909 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1911 /* Update vectorial force */
1912 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1913 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1914 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1916 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1917 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1918 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1922 gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1924 /* Inner loop uses 328 flops */
1927 /* End of innermost loop */
1929 gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1930 f+i_coord_offset,fshift+i_shift_offset);
1932 /* Increment number of inner iterations */
1933 inneriter += j_index_end - j_index_start;
1935 /* Outer loop uses 18 flops */
1938 /* Increment number of outer iterations */
1941 /* Update outer/inner flops */
1943 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*328);