2 * Note: this file was generated by the Gromacs avx_128_fma_double kernel generator.
4 * This source code is part of
8 * Copyright (c) 2001-2012, The GROMACS Development Team
10 * Gromacs is a library for molecular simulation and trajectory analysis,
11 * written by Erik Lindahl, David van der Spoel, Berk Hess, and others - for
12 * a full list of developers and information, check out http://www.gromacs.org
14 * This program is free software; you can redistribute it and/or modify it under
15 * the terms of the GNU Lesser General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option) any
19 * To help fund GROMACS development, we humbly ask that you cite
20 * the papers people have written on it - you can find them on the website.
28 #include "../nb_kernel.h"
29 #include "types/simple.h"
33 #include "gmx_math_x86_avx_128_fma_double.h"
34 #include "kernelutil_x86_avx_128_fma_double.h"
37 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwLJ_GeomW3W3_VF_avx_128_fma_double
38 * Electrostatics interaction: Coulomb
39 * VdW interaction: LennardJones
40 * Geometry: Water3-Water3
41 * Calculate force/pot: PotentialAndForce
44 nb_kernel_ElecCoul_VdwLJ_GeomW3W3_VF_avx_128_fma_double
45 (t_nblist * gmx_restrict nlist,
46 rvec * gmx_restrict xx,
47 rvec * gmx_restrict ff,
48 t_forcerec * gmx_restrict fr,
49 t_mdatoms * gmx_restrict mdatoms,
50 nb_kernel_data_t * gmx_restrict kernel_data,
51 t_nrnb * gmx_restrict nrnb)
53 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
54 * just 0 for non-waters.
55 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
56 * jnr indices corresponding to data put in the four positions in the SIMD register.
58 int i_shift_offset,i_coord_offset,outeriter,inneriter;
59 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
61 int j_coord_offsetA,j_coord_offsetB;
62 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
64 real *shiftvec,*fshift,*x,*f;
65 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
67 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
69 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
71 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
72 int vdwjidx0A,vdwjidx0B;
73 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
74 int vdwjidx1A,vdwjidx1B;
75 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
76 int vdwjidx2A,vdwjidx2B;
77 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
78 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
79 __m128d dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
80 __m128d dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
81 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
82 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
83 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
84 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
85 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
86 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
87 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
90 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
93 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
94 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
95 __m128d dummy_mask,cutoff_mask;
96 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
97 __m128d one = _mm_set1_pd(1.0);
98 __m128d two = _mm_set1_pd(2.0);
104 jindex = nlist->jindex;
106 shiftidx = nlist->shift;
108 shiftvec = fr->shift_vec[0];
109 fshift = fr->fshift[0];
110 facel = _mm_set1_pd(fr->epsfac);
111 charge = mdatoms->chargeA;
112 nvdwtype = fr->ntype;
114 vdwtype = mdatoms->typeA;
116 /* Setup water-specific parameters */
117 inr = nlist->iinr[0];
118 iq0 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
119 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
120 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
121 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
123 jq0 = _mm_set1_pd(charge[inr+0]);
124 jq1 = _mm_set1_pd(charge[inr+1]);
125 jq2 = _mm_set1_pd(charge[inr+2]);
126 vdwjidx0A = 2*vdwtype[inr+0];
127 qq00 = _mm_mul_pd(iq0,jq0);
128 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
129 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
130 qq01 = _mm_mul_pd(iq0,jq1);
131 qq02 = _mm_mul_pd(iq0,jq2);
132 qq10 = _mm_mul_pd(iq1,jq0);
133 qq11 = _mm_mul_pd(iq1,jq1);
134 qq12 = _mm_mul_pd(iq1,jq2);
135 qq20 = _mm_mul_pd(iq2,jq0);
136 qq21 = _mm_mul_pd(iq2,jq1);
137 qq22 = _mm_mul_pd(iq2,jq2);
139 /* Avoid stupid compiler warnings */
147 /* Start outer loop over neighborlists */
148 for(iidx=0; iidx<nri; iidx++)
150 /* Load shift vector for this list */
151 i_shift_offset = DIM*shiftidx[iidx];
153 /* Load limits for loop over neighbors */
154 j_index_start = jindex[iidx];
155 j_index_end = jindex[iidx+1];
157 /* Get outer coordinate index */
159 i_coord_offset = DIM*inr;
161 /* Load i particle coords and add shift vector */
162 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
163 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
165 fix0 = _mm_setzero_pd();
166 fiy0 = _mm_setzero_pd();
167 fiz0 = _mm_setzero_pd();
168 fix1 = _mm_setzero_pd();
169 fiy1 = _mm_setzero_pd();
170 fiz1 = _mm_setzero_pd();
171 fix2 = _mm_setzero_pd();
172 fiy2 = _mm_setzero_pd();
173 fiz2 = _mm_setzero_pd();
175 /* Reset potential sums */
176 velecsum = _mm_setzero_pd();
177 vvdwsum = _mm_setzero_pd();
179 /* Start inner kernel loop */
180 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
183 /* Get j neighbor index, and coordinate index */
186 j_coord_offsetA = DIM*jnrA;
187 j_coord_offsetB = DIM*jnrB;
189 /* load j atom coordinates */
190 gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
191 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
193 /* Calculate displacement vector */
194 dx00 = _mm_sub_pd(ix0,jx0);
195 dy00 = _mm_sub_pd(iy0,jy0);
196 dz00 = _mm_sub_pd(iz0,jz0);
197 dx01 = _mm_sub_pd(ix0,jx1);
198 dy01 = _mm_sub_pd(iy0,jy1);
199 dz01 = _mm_sub_pd(iz0,jz1);
200 dx02 = _mm_sub_pd(ix0,jx2);
201 dy02 = _mm_sub_pd(iy0,jy2);
202 dz02 = _mm_sub_pd(iz0,jz2);
203 dx10 = _mm_sub_pd(ix1,jx0);
204 dy10 = _mm_sub_pd(iy1,jy0);
205 dz10 = _mm_sub_pd(iz1,jz0);
206 dx11 = _mm_sub_pd(ix1,jx1);
207 dy11 = _mm_sub_pd(iy1,jy1);
208 dz11 = _mm_sub_pd(iz1,jz1);
209 dx12 = _mm_sub_pd(ix1,jx2);
210 dy12 = _mm_sub_pd(iy1,jy2);
211 dz12 = _mm_sub_pd(iz1,jz2);
212 dx20 = _mm_sub_pd(ix2,jx0);
213 dy20 = _mm_sub_pd(iy2,jy0);
214 dz20 = _mm_sub_pd(iz2,jz0);
215 dx21 = _mm_sub_pd(ix2,jx1);
216 dy21 = _mm_sub_pd(iy2,jy1);
217 dz21 = _mm_sub_pd(iz2,jz1);
218 dx22 = _mm_sub_pd(ix2,jx2);
219 dy22 = _mm_sub_pd(iy2,jy2);
220 dz22 = _mm_sub_pd(iz2,jz2);
222 /* Calculate squared distance and things based on it */
223 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
224 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
225 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
226 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
227 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
228 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
229 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
230 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
231 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
233 rinv00 = gmx_mm_invsqrt_pd(rsq00);
234 rinv01 = gmx_mm_invsqrt_pd(rsq01);
235 rinv02 = gmx_mm_invsqrt_pd(rsq02);
236 rinv10 = gmx_mm_invsqrt_pd(rsq10);
237 rinv11 = gmx_mm_invsqrt_pd(rsq11);
238 rinv12 = gmx_mm_invsqrt_pd(rsq12);
239 rinv20 = gmx_mm_invsqrt_pd(rsq20);
240 rinv21 = gmx_mm_invsqrt_pd(rsq21);
241 rinv22 = gmx_mm_invsqrt_pd(rsq22);
243 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
244 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
245 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
246 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
247 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
248 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
249 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
250 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
251 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
253 fjx0 = _mm_setzero_pd();
254 fjy0 = _mm_setzero_pd();
255 fjz0 = _mm_setzero_pd();
256 fjx1 = _mm_setzero_pd();
257 fjy1 = _mm_setzero_pd();
258 fjz1 = _mm_setzero_pd();
259 fjx2 = _mm_setzero_pd();
260 fjy2 = _mm_setzero_pd();
261 fjz2 = _mm_setzero_pd();
263 /**************************
264 * CALCULATE INTERACTIONS *
265 **************************/
267 /* COULOMB ELECTROSTATICS */
268 velec = _mm_mul_pd(qq00,rinv00);
269 felec = _mm_mul_pd(velec,rinvsq00);
271 /* LENNARD-JONES DISPERSION/REPULSION */
273 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
274 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
275 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
276 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
277 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
279 /* Update potential sum for this i atom from the interaction with this j atom. */
280 velecsum = _mm_add_pd(velecsum,velec);
281 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
283 fscal = _mm_add_pd(felec,fvdw);
285 /* Update vectorial force */
286 fix0 = _mm_macc_pd(dx00,fscal,fix0);
287 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
288 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
290 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
291 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
292 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
294 /**************************
295 * CALCULATE INTERACTIONS *
296 **************************/
298 /* COULOMB ELECTROSTATICS */
299 velec = _mm_mul_pd(qq01,rinv01);
300 felec = _mm_mul_pd(velec,rinvsq01);
302 /* Update potential sum for this i atom from the interaction with this j atom. */
303 velecsum = _mm_add_pd(velecsum,velec);
307 /* Update vectorial force */
308 fix0 = _mm_macc_pd(dx01,fscal,fix0);
309 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
310 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
312 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
313 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
314 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
316 /**************************
317 * CALCULATE INTERACTIONS *
318 **************************/
320 /* COULOMB ELECTROSTATICS */
321 velec = _mm_mul_pd(qq02,rinv02);
322 felec = _mm_mul_pd(velec,rinvsq02);
324 /* Update potential sum for this i atom from the interaction with this j atom. */
325 velecsum = _mm_add_pd(velecsum,velec);
329 /* Update vectorial force */
330 fix0 = _mm_macc_pd(dx02,fscal,fix0);
331 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
332 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
334 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
335 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
336 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
338 /**************************
339 * CALCULATE INTERACTIONS *
340 **************************/
342 /* COULOMB ELECTROSTATICS */
343 velec = _mm_mul_pd(qq10,rinv10);
344 felec = _mm_mul_pd(velec,rinvsq10);
346 /* Update potential sum for this i atom from the interaction with this j atom. */
347 velecsum = _mm_add_pd(velecsum,velec);
351 /* Update vectorial force */
352 fix1 = _mm_macc_pd(dx10,fscal,fix1);
353 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
354 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
356 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
357 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
358 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
360 /**************************
361 * CALCULATE INTERACTIONS *
362 **************************/
364 /* COULOMB ELECTROSTATICS */
365 velec = _mm_mul_pd(qq11,rinv11);
366 felec = _mm_mul_pd(velec,rinvsq11);
368 /* Update potential sum for this i atom from the interaction with this j atom. */
369 velecsum = _mm_add_pd(velecsum,velec);
373 /* Update vectorial force */
374 fix1 = _mm_macc_pd(dx11,fscal,fix1);
375 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
376 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
378 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
379 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
380 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
382 /**************************
383 * CALCULATE INTERACTIONS *
384 **************************/
386 /* COULOMB ELECTROSTATICS */
387 velec = _mm_mul_pd(qq12,rinv12);
388 felec = _mm_mul_pd(velec,rinvsq12);
390 /* Update potential sum for this i atom from the interaction with this j atom. */
391 velecsum = _mm_add_pd(velecsum,velec);
395 /* Update vectorial force */
396 fix1 = _mm_macc_pd(dx12,fscal,fix1);
397 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
398 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
400 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
401 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
402 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
404 /**************************
405 * CALCULATE INTERACTIONS *
406 **************************/
408 /* COULOMB ELECTROSTATICS */
409 velec = _mm_mul_pd(qq20,rinv20);
410 felec = _mm_mul_pd(velec,rinvsq20);
412 /* Update potential sum for this i atom from the interaction with this j atom. */
413 velecsum = _mm_add_pd(velecsum,velec);
417 /* Update vectorial force */
418 fix2 = _mm_macc_pd(dx20,fscal,fix2);
419 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
420 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
422 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
423 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
424 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
426 /**************************
427 * CALCULATE INTERACTIONS *
428 **************************/
430 /* COULOMB ELECTROSTATICS */
431 velec = _mm_mul_pd(qq21,rinv21);
432 felec = _mm_mul_pd(velec,rinvsq21);
434 /* Update potential sum for this i atom from the interaction with this j atom. */
435 velecsum = _mm_add_pd(velecsum,velec);
439 /* Update vectorial force */
440 fix2 = _mm_macc_pd(dx21,fscal,fix2);
441 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
442 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
444 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
445 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
446 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
448 /**************************
449 * CALCULATE INTERACTIONS *
450 **************************/
452 /* COULOMB ELECTROSTATICS */
453 velec = _mm_mul_pd(qq22,rinv22);
454 felec = _mm_mul_pd(velec,rinvsq22);
456 /* Update potential sum for this i atom from the interaction with this j atom. */
457 velecsum = _mm_add_pd(velecsum,velec);
461 /* Update vectorial force */
462 fix2 = _mm_macc_pd(dx22,fscal,fix2);
463 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
464 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
466 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
467 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
468 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
470 gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
472 /* Inner loop uses 291 flops */
479 j_coord_offsetA = DIM*jnrA;
481 /* load j atom coordinates */
482 gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
483 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
485 /* Calculate displacement vector */
486 dx00 = _mm_sub_pd(ix0,jx0);
487 dy00 = _mm_sub_pd(iy0,jy0);
488 dz00 = _mm_sub_pd(iz0,jz0);
489 dx01 = _mm_sub_pd(ix0,jx1);
490 dy01 = _mm_sub_pd(iy0,jy1);
491 dz01 = _mm_sub_pd(iz0,jz1);
492 dx02 = _mm_sub_pd(ix0,jx2);
493 dy02 = _mm_sub_pd(iy0,jy2);
494 dz02 = _mm_sub_pd(iz0,jz2);
495 dx10 = _mm_sub_pd(ix1,jx0);
496 dy10 = _mm_sub_pd(iy1,jy0);
497 dz10 = _mm_sub_pd(iz1,jz0);
498 dx11 = _mm_sub_pd(ix1,jx1);
499 dy11 = _mm_sub_pd(iy1,jy1);
500 dz11 = _mm_sub_pd(iz1,jz1);
501 dx12 = _mm_sub_pd(ix1,jx2);
502 dy12 = _mm_sub_pd(iy1,jy2);
503 dz12 = _mm_sub_pd(iz1,jz2);
504 dx20 = _mm_sub_pd(ix2,jx0);
505 dy20 = _mm_sub_pd(iy2,jy0);
506 dz20 = _mm_sub_pd(iz2,jz0);
507 dx21 = _mm_sub_pd(ix2,jx1);
508 dy21 = _mm_sub_pd(iy2,jy1);
509 dz21 = _mm_sub_pd(iz2,jz1);
510 dx22 = _mm_sub_pd(ix2,jx2);
511 dy22 = _mm_sub_pd(iy2,jy2);
512 dz22 = _mm_sub_pd(iz2,jz2);
514 /* Calculate squared distance and things based on it */
515 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
516 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
517 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
518 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
519 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
520 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
521 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
522 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
523 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
525 rinv00 = gmx_mm_invsqrt_pd(rsq00);
526 rinv01 = gmx_mm_invsqrt_pd(rsq01);
527 rinv02 = gmx_mm_invsqrt_pd(rsq02);
528 rinv10 = gmx_mm_invsqrt_pd(rsq10);
529 rinv11 = gmx_mm_invsqrt_pd(rsq11);
530 rinv12 = gmx_mm_invsqrt_pd(rsq12);
531 rinv20 = gmx_mm_invsqrt_pd(rsq20);
532 rinv21 = gmx_mm_invsqrt_pd(rsq21);
533 rinv22 = gmx_mm_invsqrt_pd(rsq22);
535 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
536 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
537 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
538 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
539 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
540 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
541 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
542 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
543 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
545 fjx0 = _mm_setzero_pd();
546 fjy0 = _mm_setzero_pd();
547 fjz0 = _mm_setzero_pd();
548 fjx1 = _mm_setzero_pd();
549 fjy1 = _mm_setzero_pd();
550 fjz1 = _mm_setzero_pd();
551 fjx2 = _mm_setzero_pd();
552 fjy2 = _mm_setzero_pd();
553 fjz2 = _mm_setzero_pd();
555 /**************************
556 * CALCULATE INTERACTIONS *
557 **************************/
559 /* COULOMB ELECTROSTATICS */
560 velec = _mm_mul_pd(qq00,rinv00);
561 felec = _mm_mul_pd(velec,rinvsq00);
563 /* LENNARD-JONES DISPERSION/REPULSION */
565 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
566 vvdw6 = _mm_mul_pd(c6_00,rinvsix);
567 vvdw12 = _mm_mul_pd(c12_00,_mm_mul_pd(rinvsix,rinvsix));
568 vvdw = _mm_msub_pd( vvdw12,one_twelfth, _mm_mul_pd(vvdw6,one_sixth) );
569 fvdw = _mm_mul_pd(_mm_sub_pd(vvdw12,vvdw6),rinvsq00);
571 /* Update potential sum for this i atom from the interaction with this j atom. */
572 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
573 velecsum = _mm_add_pd(velecsum,velec);
574 vvdw = _mm_unpacklo_pd(vvdw,_mm_setzero_pd());
575 vvdwsum = _mm_add_pd(vvdwsum,vvdw);
577 fscal = _mm_add_pd(felec,fvdw);
579 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
581 /* Update vectorial force */
582 fix0 = _mm_macc_pd(dx00,fscal,fix0);
583 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
584 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
586 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
587 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
588 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
590 /**************************
591 * CALCULATE INTERACTIONS *
592 **************************/
594 /* COULOMB ELECTROSTATICS */
595 velec = _mm_mul_pd(qq01,rinv01);
596 felec = _mm_mul_pd(velec,rinvsq01);
598 /* Update potential sum for this i atom from the interaction with this j atom. */
599 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
600 velecsum = _mm_add_pd(velecsum,velec);
604 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
606 /* Update vectorial force */
607 fix0 = _mm_macc_pd(dx01,fscal,fix0);
608 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
609 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
611 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
612 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
613 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
615 /**************************
616 * CALCULATE INTERACTIONS *
617 **************************/
619 /* COULOMB ELECTROSTATICS */
620 velec = _mm_mul_pd(qq02,rinv02);
621 felec = _mm_mul_pd(velec,rinvsq02);
623 /* Update potential sum for this i atom from the interaction with this j atom. */
624 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
625 velecsum = _mm_add_pd(velecsum,velec);
629 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
631 /* Update vectorial force */
632 fix0 = _mm_macc_pd(dx02,fscal,fix0);
633 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
634 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
636 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
637 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
638 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
640 /**************************
641 * CALCULATE INTERACTIONS *
642 **************************/
644 /* COULOMB ELECTROSTATICS */
645 velec = _mm_mul_pd(qq10,rinv10);
646 felec = _mm_mul_pd(velec,rinvsq10);
648 /* Update potential sum for this i atom from the interaction with this j atom. */
649 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
650 velecsum = _mm_add_pd(velecsum,velec);
654 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
656 /* Update vectorial force */
657 fix1 = _mm_macc_pd(dx10,fscal,fix1);
658 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
659 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
661 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
662 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
663 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
665 /**************************
666 * CALCULATE INTERACTIONS *
667 **************************/
669 /* COULOMB ELECTROSTATICS */
670 velec = _mm_mul_pd(qq11,rinv11);
671 felec = _mm_mul_pd(velec,rinvsq11);
673 /* Update potential sum for this i atom from the interaction with this j atom. */
674 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
675 velecsum = _mm_add_pd(velecsum,velec);
679 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
681 /* Update vectorial force */
682 fix1 = _mm_macc_pd(dx11,fscal,fix1);
683 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
684 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
686 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
687 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
688 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
690 /**************************
691 * CALCULATE INTERACTIONS *
692 **************************/
694 /* COULOMB ELECTROSTATICS */
695 velec = _mm_mul_pd(qq12,rinv12);
696 felec = _mm_mul_pd(velec,rinvsq12);
698 /* Update potential sum for this i atom from the interaction with this j atom. */
699 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
700 velecsum = _mm_add_pd(velecsum,velec);
704 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
706 /* Update vectorial force */
707 fix1 = _mm_macc_pd(dx12,fscal,fix1);
708 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
709 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
711 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
712 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
713 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
715 /**************************
716 * CALCULATE INTERACTIONS *
717 **************************/
719 /* COULOMB ELECTROSTATICS */
720 velec = _mm_mul_pd(qq20,rinv20);
721 felec = _mm_mul_pd(velec,rinvsq20);
723 /* Update potential sum for this i atom from the interaction with this j atom. */
724 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
725 velecsum = _mm_add_pd(velecsum,velec);
729 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
731 /* Update vectorial force */
732 fix2 = _mm_macc_pd(dx20,fscal,fix2);
733 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
734 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
736 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
737 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
738 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
740 /**************************
741 * CALCULATE INTERACTIONS *
742 **************************/
744 /* COULOMB ELECTROSTATICS */
745 velec = _mm_mul_pd(qq21,rinv21);
746 felec = _mm_mul_pd(velec,rinvsq21);
748 /* Update potential sum for this i atom from the interaction with this j atom. */
749 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
750 velecsum = _mm_add_pd(velecsum,velec);
754 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
756 /* Update vectorial force */
757 fix2 = _mm_macc_pd(dx21,fscal,fix2);
758 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
759 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
761 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
762 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
763 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
765 /**************************
766 * CALCULATE INTERACTIONS *
767 **************************/
769 /* COULOMB ELECTROSTATICS */
770 velec = _mm_mul_pd(qq22,rinv22);
771 felec = _mm_mul_pd(velec,rinvsq22);
773 /* Update potential sum for this i atom from the interaction with this j atom. */
774 velec = _mm_unpacklo_pd(velec,_mm_setzero_pd());
775 velecsum = _mm_add_pd(velecsum,velec);
779 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
781 /* Update vectorial force */
782 fix2 = _mm_macc_pd(dx22,fscal,fix2);
783 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
784 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
786 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
787 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
788 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
790 gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
792 /* Inner loop uses 291 flops */
795 /* End of innermost loop */
797 gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
798 f+i_coord_offset,fshift+i_shift_offset);
801 /* Update potential energies */
802 gmx_mm_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
803 gmx_mm_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
805 /* Increment number of inner iterations */
806 inneriter += j_index_end - j_index_start;
808 /* Outer loop uses 20 flops */
811 /* Increment number of outer iterations */
814 /* Update outer/inner flops */
816 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*291);
819 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwLJ_GeomW3W3_F_avx_128_fma_double
820 * Electrostatics interaction: Coulomb
821 * VdW interaction: LennardJones
822 * Geometry: Water3-Water3
823 * Calculate force/pot: Force
826 nb_kernel_ElecCoul_VdwLJ_GeomW3W3_F_avx_128_fma_double
827 (t_nblist * gmx_restrict nlist,
828 rvec * gmx_restrict xx,
829 rvec * gmx_restrict ff,
830 t_forcerec * gmx_restrict fr,
831 t_mdatoms * gmx_restrict mdatoms,
832 nb_kernel_data_t * gmx_restrict kernel_data,
833 t_nrnb * gmx_restrict nrnb)
835 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
836 * just 0 for non-waters.
837 * Suffixes A,B refer to j loop unrolling done with SSE double precision, e.g. for the two different
838 * jnr indices corresponding to data put in the four positions in the SIMD register.
840 int i_shift_offset,i_coord_offset,outeriter,inneriter;
841 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
843 int j_coord_offsetA,j_coord_offsetB;
844 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
846 real *shiftvec,*fshift,*x,*f;
847 __m128d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
849 __m128d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
851 __m128d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
853 __m128d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
854 int vdwjidx0A,vdwjidx0B;
855 __m128d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
856 int vdwjidx1A,vdwjidx1B;
857 __m128d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
858 int vdwjidx2A,vdwjidx2B;
859 __m128d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
860 __m128d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
861 __m128d dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
862 __m128d dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
863 __m128d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
864 __m128d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
865 __m128d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
866 __m128d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
867 __m128d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
868 __m128d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
869 __m128d velec,felec,velecsum,facel,crf,krf,krf2;
872 __m128d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
875 __m128d one_sixth = _mm_set1_pd(1.0/6.0);
876 __m128d one_twelfth = _mm_set1_pd(1.0/12.0);
877 __m128d dummy_mask,cutoff_mask;
878 __m128d signbit = gmx_mm_castsi128_pd( _mm_set_epi32(0x80000000,0x00000000,0x80000000,0x00000000) );
879 __m128d one = _mm_set1_pd(1.0);
880 __m128d two = _mm_set1_pd(2.0);
886 jindex = nlist->jindex;
888 shiftidx = nlist->shift;
890 shiftvec = fr->shift_vec[0];
891 fshift = fr->fshift[0];
892 facel = _mm_set1_pd(fr->epsfac);
893 charge = mdatoms->chargeA;
894 nvdwtype = fr->ntype;
896 vdwtype = mdatoms->typeA;
898 /* Setup water-specific parameters */
899 inr = nlist->iinr[0];
900 iq0 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+0]));
901 iq1 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+1]));
902 iq2 = _mm_mul_pd(facel,_mm_set1_pd(charge[inr+2]));
903 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
905 jq0 = _mm_set1_pd(charge[inr+0]);
906 jq1 = _mm_set1_pd(charge[inr+1]);
907 jq2 = _mm_set1_pd(charge[inr+2]);
908 vdwjidx0A = 2*vdwtype[inr+0];
909 qq00 = _mm_mul_pd(iq0,jq0);
910 c6_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A]);
911 c12_00 = _mm_set1_pd(vdwparam[vdwioffset0+vdwjidx0A+1]);
912 qq01 = _mm_mul_pd(iq0,jq1);
913 qq02 = _mm_mul_pd(iq0,jq2);
914 qq10 = _mm_mul_pd(iq1,jq0);
915 qq11 = _mm_mul_pd(iq1,jq1);
916 qq12 = _mm_mul_pd(iq1,jq2);
917 qq20 = _mm_mul_pd(iq2,jq0);
918 qq21 = _mm_mul_pd(iq2,jq1);
919 qq22 = _mm_mul_pd(iq2,jq2);
921 /* Avoid stupid compiler warnings */
929 /* Start outer loop over neighborlists */
930 for(iidx=0; iidx<nri; iidx++)
932 /* Load shift vector for this list */
933 i_shift_offset = DIM*shiftidx[iidx];
935 /* Load limits for loop over neighbors */
936 j_index_start = jindex[iidx];
937 j_index_end = jindex[iidx+1];
939 /* Get outer coordinate index */
941 i_coord_offset = DIM*inr;
943 /* Load i particle coords and add shift vector */
944 gmx_mm_load_shift_and_3rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
945 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
947 fix0 = _mm_setzero_pd();
948 fiy0 = _mm_setzero_pd();
949 fiz0 = _mm_setzero_pd();
950 fix1 = _mm_setzero_pd();
951 fiy1 = _mm_setzero_pd();
952 fiz1 = _mm_setzero_pd();
953 fix2 = _mm_setzero_pd();
954 fiy2 = _mm_setzero_pd();
955 fiz2 = _mm_setzero_pd();
957 /* Start inner kernel loop */
958 for(jidx=j_index_start; jidx<j_index_end-1; jidx+=2)
961 /* Get j neighbor index, and coordinate index */
964 j_coord_offsetA = DIM*jnrA;
965 j_coord_offsetB = DIM*jnrB;
967 /* load j atom coordinates */
968 gmx_mm_load_3rvec_2ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
969 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
971 /* Calculate displacement vector */
972 dx00 = _mm_sub_pd(ix0,jx0);
973 dy00 = _mm_sub_pd(iy0,jy0);
974 dz00 = _mm_sub_pd(iz0,jz0);
975 dx01 = _mm_sub_pd(ix0,jx1);
976 dy01 = _mm_sub_pd(iy0,jy1);
977 dz01 = _mm_sub_pd(iz0,jz1);
978 dx02 = _mm_sub_pd(ix0,jx2);
979 dy02 = _mm_sub_pd(iy0,jy2);
980 dz02 = _mm_sub_pd(iz0,jz2);
981 dx10 = _mm_sub_pd(ix1,jx0);
982 dy10 = _mm_sub_pd(iy1,jy0);
983 dz10 = _mm_sub_pd(iz1,jz0);
984 dx11 = _mm_sub_pd(ix1,jx1);
985 dy11 = _mm_sub_pd(iy1,jy1);
986 dz11 = _mm_sub_pd(iz1,jz1);
987 dx12 = _mm_sub_pd(ix1,jx2);
988 dy12 = _mm_sub_pd(iy1,jy2);
989 dz12 = _mm_sub_pd(iz1,jz2);
990 dx20 = _mm_sub_pd(ix2,jx0);
991 dy20 = _mm_sub_pd(iy2,jy0);
992 dz20 = _mm_sub_pd(iz2,jz0);
993 dx21 = _mm_sub_pd(ix2,jx1);
994 dy21 = _mm_sub_pd(iy2,jy1);
995 dz21 = _mm_sub_pd(iz2,jz1);
996 dx22 = _mm_sub_pd(ix2,jx2);
997 dy22 = _mm_sub_pd(iy2,jy2);
998 dz22 = _mm_sub_pd(iz2,jz2);
1000 /* Calculate squared distance and things based on it */
1001 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1002 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1003 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1004 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1005 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1006 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1007 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1008 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1009 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1011 rinv00 = gmx_mm_invsqrt_pd(rsq00);
1012 rinv01 = gmx_mm_invsqrt_pd(rsq01);
1013 rinv02 = gmx_mm_invsqrt_pd(rsq02);
1014 rinv10 = gmx_mm_invsqrt_pd(rsq10);
1015 rinv11 = gmx_mm_invsqrt_pd(rsq11);
1016 rinv12 = gmx_mm_invsqrt_pd(rsq12);
1017 rinv20 = gmx_mm_invsqrt_pd(rsq20);
1018 rinv21 = gmx_mm_invsqrt_pd(rsq21);
1019 rinv22 = gmx_mm_invsqrt_pd(rsq22);
1021 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
1022 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
1023 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
1024 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
1025 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
1026 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
1027 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
1028 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
1029 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
1031 fjx0 = _mm_setzero_pd();
1032 fjy0 = _mm_setzero_pd();
1033 fjz0 = _mm_setzero_pd();
1034 fjx1 = _mm_setzero_pd();
1035 fjy1 = _mm_setzero_pd();
1036 fjz1 = _mm_setzero_pd();
1037 fjx2 = _mm_setzero_pd();
1038 fjy2 = _mm_setzero_pd();
1039 fjz2 = _mm_setzero_pd();
1041 /**************************
1042 * CALCULATE INTERACTIONS *
1043 **************************/
1045 /* COULOMB ELECTROSTATICS */
1046 velec = _mm_mul_pd(qq00,rinv00);
1047 felec = _mm_mul_pd(velec,rinvsq00);
1049 /* LENNARD-JONES DISPERSION/REPULSION */
1051 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1052 fvdw = _mm_mul_pd(_mm_msub_pd(c12_00,rinvsix,c6_00),_mm_mul_pd(rinvsix,rinvsq00));
1054 fscal = _mm_add_pd(felec,fvdw);
1056 /* Update vectorial force */
1057 fix0 = _mm_macc_pd(dx00,fscal,fix0);
1058 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
1059 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
1061 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
1062 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
1063 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
1065 /**************************
1066 * CALCULATE INTERACTIONS *
1067 **************************/
1069 /* COULOMB ELECTROSTATICS */
1070 velec = _mm_mul_pd(qq01,rinv01);
1071 felec = _mm_mul_pd(velec,rinvsq01);
1075 /* Update vectorial force */
1076 fix0 = _mm_macc_pd(dx01,fscal,fix0);
1077 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
1078 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
1080 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
1081 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
1082 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
1084 /**************************
1085 * CALCULATE INTERACTIONS *
1086 **************************/
1088 /* COULOMB ELECTROSTATICS */
1089 velec = _mm_mul_pd(qq02,rinv02);
1090 felec = _mm_mul_pd(velec,rinvsq02);
1094 /* Update vectorial force */
1095 fix0 = _mm_macc_pd(dx02,fscal,fix0);
1096 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
1097 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
1099 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
1100 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
1101 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
1103 /**************************
1104 * CALCULATE INTERACTIONS *
1105 **************************/
1107 /* COULOMB ELECTROSTATICS */
1108 velec = _mm_mul_pd(qq10,rinv10);
1109 felec = _mm_mul_pd(velec,rinvsq10);
1113 /* Update vectorial force */
1114 fix1 = _mm_macc_pd(dx10,fscal,fix1);
1115 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
1116 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
1118 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
1119 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
1120 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
1122 /**************************
1123 * CALCULATE INTERACTIONS *
1124 **************************/
1126 /* COULOMB ELECTROSTATICS */
1127 velec = _mm_mul_pd(qq11,rinv11);
1128 felec = _mm_mul_pd(velec,rinvsq11);
1132 /* Update vectorial force */
1133 fix1 = _mm_macc_pd(dx11,fscal,fix1);
1134 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
1135 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
1137 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
1138 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
1139 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
1141 /**************************
1142 * CALCULATE INTERACTIONS *
1143 **************************/
1145 /* COULOMB ELECTROSTATICS */
1146 velec = _mm_mul_pd(qq12,rinv12);
1147 felec = _mm_mul_pd(velec,rinvsq12);
1151 /* Update vectorial force */
1152 fix1 = _mm_macc_pd(dx12,fscal,fix1);
1153 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
1154 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1156 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1157 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1158 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1160 /**************************
1161 * CALCULATE INTERACTIONS *
1162 **************************/
1164 /* COULOMB ELECTROSTATICS */
1165 velec = _mm_mul_pd(qq20,rinv20);
1166 felec = _mm_mul_pd(velec,rinvsq20);
1170 /* Update vectorial force */
1171 fix2 = _mm_macc_pd(dx20,fscal,fix2);
1172 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
1173 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
1175 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
1176 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
1177 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
1179 /**************************
1180 * CALCULATE INTERACTIONS *
1181 **************************/
1183 /* COULOMB ELECTROSTATICS */
1184 velec = _mm_mul_pd(qq21,rinv21);
1185 felec = _mm_mul_pd(velec,rinvsq21);
1189 /* Update vectorial force */
1190 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1191 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1192 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1194 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1195 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1196 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1198 /**************************
1199 * CALCULATE INTERACTIONS *
1200 **************************/
1202 /* COULOMB ELECTROSTATICS */
1203 velec = _mm_mul_pd(qq22,rinv22);
1204 felec = _mm_mul_pd(velec,rinvsq22);
1208 /* Update vectorial force */
1209 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1210 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1211 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1213 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1214 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1215 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1217 gmx_mm_decrement_3rvec_2ptr_swizzle_pd(f+j_coord_offsetA,f+j_coord_offsetB,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1219 /* Inner loop uses 277 flops */
1222 if(jidx<j_index_end)
1226 j_coord_offsetA = DIM*jnrA;
1228 /* load j atom coordinates */
1229 gmx_mm_load_3rvec_1ptr_swizzle_pd(x+j_coord_offsetA,
1230 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1232 /* Calculate displacement vector */
1233 dx00 = _mm_sub_pd(ix0,jx0);
1234 dy00 = _mm_sub_pd(iy0,jy0);
1235 dz00 = _mm_sub_pd(iz0,jz0);
1236 dx01 = _mm_sub_pd(ix0,jx1);
1237 dy01 = _mm_sub_pd(iy0,jy1);
1238 dz01 = _mm_sub_pd(iz0,jz1);
1239 dx02 = _mm_sub_pd(ix0,jx2);
1240 dy02 = _mm_sub_pd(iy0,jy2);
1241 dz02 = _mm_sub_pd(iz0,jz2);
1242 dx10 = _mm_sub_pd(ix1,jx0);
1243 dy10 = _mm_sub_pd(iy1,jy0);
1244 dz10 = _mm_sub_pd(iz1,jz0);
1245 dx11 = _mm_sub_pd(ix1,jx1);
1246 dy11 = _mm_sub_pd(iy1,jy1);
1247 dz11 = _mm_sub_pd(iz1,jz1);
1248 dx12 = _mm_sub_pd(ix1,jx2);
1249 dy12 = _mm_sub_pd(iy1,jy2);
1250 dz12 = _mm_sub_pd(iz1,jz2);
1251 dx20 = _mm_sub_pd(ix2,jx0);
1252 dy20 = _mm_sub_pd(iy2,jy0);
1253 dz20 = _mm_sub_pd(iz2,jz0);
1254 dx21 = _mm_sub_pd(ix2,jx1);
1255 dy21 = _mm_sub_pd(iy2,jy1);
1256 dz21 = _mm_sub_pd(iz2,jz1);
1257 dx22 = _mm_sub_pd(ix2,jx2);
1258 dy22 = _mm_sub_pd(iy2,jy2);
1259 dz22 = _mm_sub_pd(iz2,jz2);
1261 /* Calculate squared distance and things based on it */
1262 rsq00 = gmx_mm_calc_rsq_pd(dx00,dy00,dz00);
1263 rsq01 = gmx_mm_calc_rsq_pd(dx01,dy01,dz01);
1264 rsq02 = gmx_mm_calc_rsq_pd(dx02,dy02,dz02);
1265 rsq10 = gmx_mm_calc_rsq_pd(dx10,dy10,dz10);
1266 rsq11 = gmx_mm_calc_rsq_pd(dx11,dy11,dz11);
1267 rsq12 = gmx_mm_calc_rsq_pd(dx12,dy12,dz12);
1268 rsq20 = gmx_mm_calc_rsq_pd(dx20,dy20,dz20);
1269 rsq21 = gmx_mm_calc_rsq_pd(dx21,dy21,dz21);
1270 rsq22 = gmx_mm_calc_rsq_pd(dx22,dy22,dz22);
1272 rinv00 = gmx_mm_invsqrt_pd(rsq00);
1273 rinv01 = gmx_mm_invsqrt_pd(rsq01);
1274 rinv02 = gmx_mm_invsqrt_pd(rsq02);
1275 rinv10 = gmx_mm_invsqrt_pd(rsq10);
1276 rinv11 = gmx_mm_invsqrt_pd(rsq11);
1277 rinv12 = gmx_mm_invsqrt_pd(rsq12);
1278 rinv20 = gmx_mm_invsqrt_pd(rsq20);
1279 rinv21 = gmx_mm_invsqrt_pd(rsq21);
1280 rinv22 = gmx_mm_invsqrt_pd(rsq22);
1282 rinvsq00 = _mm_mul_pd(rinv00,rinv00);
1283 rinvsq01 = _mm_mul_pd(rinv01,rinv01);
1284 rinvsq02 = _mm_mul_pd(rinv02,rinv02);
1285 rinvsq10 = _mm_mul_pd(rinv10,rinv10);
1286 rinvsq11 = _mm_mul_pd(rinv11,rinv11);
1287 rinvsq12 = _mm_mul_pd(rinv12,rinv12);
1288 rinvsq20 = _mm_mul_pd(rinv20,rinv20);
1289 rinvsq21 = _mm_mul_pd(rinv21,rinv21);
1290 rinvsq22 = _mm_mul_pd(rinv22,rinv22);
1292 fjx0 = _mm_setzero_pd();
1293 fjy0 = _mm_setzero_pd();
1294 fjz0 = _mm_setzero_pd();
1295 fjx1 = _mm_setzero_pd();
1296 fjy1 = _mm_setzero_pd();
1297 fjz1 = _mm_setzero_pd();
1298 fjx2 = _mm_setzero_pd();
1299 fjy2 = _mm_setzero_pd();
1300 fjz2 = _mm_setzero_pd();
1302 /**************************
1303 * CALCULATE INTERACTIONS *
1304 **************************/
1306 /* COULOMB ELECTROSTATICS */
1307 velec = _mm_mul_pd(qq00,rinv00);
1308 felec = _mm_mul_pd(velec,rinvsq00);
1310 /* LENNARD-JONES DISPERSION/REPULSION */
1312 rinvsix = _mm_mul_pd(_mm_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1313 fvdw = _mm_mul_pd(_mm_msub_pd(c12_00,rinvsix,c6_00),_mm_mul_pd(rinvsix,rinvsq00));
1315 fscal = _mm_add_pd(felec,fvdw);
1317 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1319 /* Update vectorial force */
1320 fix0 = _mm_macc_pd(dx00,fscal,fix0);
1321 fiy0 = _mm_macc_pd(dy00,fscal,fiy0);
1322 fiz0 = _mm_macc_pd(dz00,fscal,fiz0);
1324 fjx0 = _mm_macc_pd(dx00,fscal,fjx0);
1325 fjy0 = _mm_macc_pd(dy00,fscal,fjy0);
1326 fjz0 = _mm_macc_pd(dz00,fscal,fjz0);
1328 /**************************
1329 * CALCULATE INTERACTIONS *
1330 **************************/
1332 /* COULOMB ELECTROSTATICS */
1333 velec = _mm_mul_pd(qq01,rinv01);
1334 felec = _mm_mul_pd(velec,rinvsq01);
1338 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1340 /* Update vectorial force */
1341 fix0 = _mm_macc_pd(dx01,fscal,fix0);
1342 fiy0 = _mm_macc_pd(dy01,fscal,fiy0);
1343 fiz0 = _mm_macc_pd(dz01,fscal,fiz0);
1345 fjx1 = _mm_macc_pd(dx01,fscal,fjx1);
1346 fjy1 = _mm_macc_pd(dy01,fscal,fjy1);
1347 fjz1 = _mm_macc_pd(dz01,fscal,fjz1);
1349 /**************************
1350 * CALCULATE INTERACTIONS *
1351 **************************/
1353 /* COULOMB ELECTROSTATICS */
1354 velec = _mm_mul_pd(qq02,rinv02);
1355 felec = _mm_mul_pd(velec,rinvsq02);
1359 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1361 /* Update vectorial force */
1362 fix0 = _mm_macc_pd(dx02,fscal,fix0);
1363 fiy0 = _mm_macc_pd(dy02,fscal,fiy0);
1364 fiz0 = _mm_macc_pd(dz02,fscal,fiz0);
1366 fjx2 = _mm_macc_pd(dx02,fscal,fjx2);
1367 fjy2 = _mm_macc_pd(dy02,fscal,fjy2);
1368 fjz2 = _mm_macc_pd(dz02,fscal,fjz2);
1370 /**************************
1371 * CALCULATE INTERACTIONS *
1372 **************************/
1374 /* COULOMB ELECTROSTATICS */
1375 velec = _mm_mul_pd(qq10,rinv10);
1376 felec = _mm_mul_pd(velec,rinvsq10);
1380 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1382 /* Update vectorial force */
1383 fix1 = _mm_macc_pd(dx10,fscal,fix1);
1384 fiy1 = _mm_macc_pd(dy10,fscal,fiy1);
1385 fiz1 = _mm_macc_pd(dz10,fscal,fiz1);
1387 fjx0 = _mm_macc_pd(dx10,fscal,fjx0);
1388 fjy0 = _mm_macc_pd(dy10,fscal,fjy0);
1389 fjz0 = _mm_macc_pd(dz10,fscal,fjz0);
1391 /**************************
1392 * CALCULATE INTERACTIONS *
1393 **************************/
1395 /* COULOMB ELECTROSTATICS */
1396 velec = _mm_mul_pd(qq11,rinv11);
1397 felec = _mm_mul_pd(velec,rinvsq11);
1401 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1403 /* Update vectorial force */
1404 fix1 = _mm_macc_pd(dx11,fscal,fix1);
1405 fiy1 = _mm_macc_pd(dy11,fscal,fiy1);
1406 fiz1 = _mm_macc_pd(dz11,fscal,fiz1);
1408 fjx1 = _mm_macc_pd(dx11,fscal,fjx1);
1409 fjy1 = _mm_macc_pd(dy11,fscal,fjy1);
1410 fjz1 = _mm_macc_pd(dz11,fscal,fjz1);
1412 /**************************
1413 * CALCULATE INTERACTIONS *
1414 **************************/
1416 /* COULOMB ELECTROSTATICS */
1417 velec = _mm_mul_pd(qq12,rinv12);
1418 felec = _mm_mul_pd(velec,rinvsq12);
1422 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1424 /* Update vectorial force */
1425 fix1 = _mm_macc_pd(dx12,fscal,fix1);
1426 fiy1 = _mm_macc_pd(dy12,fscal,fiy1);
1427 fiz1 = _mm_macc_pd(dz12,fscal,fiz1);
1429 fjx2 = _mm_macc_pd(dx12,fscal,fjx2);
1430 fjy2 = _mm_macc_pd(dy12,fscal,fjy2);
1431 fjz2 = _mm_macc_pd(dz12,fscal,fjz2);
1433 /**************************
1434 * CALCULATE INTERACTIONS *
1435 **************************/
1437 /* COULOMB ELECTROSTATICS */
1438 velec = _mm_mul_pd(qq20,rinv20);
1439 felec = _mm_mul_pd(velec,rinvsq20);
1443 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1445 /* Update vectorial force */
1446 fix2 = _mm_macc_pd(dx20,fscal,fix2);
1447 fiy2 = _mm_macc_pd(dy20,fscal,fiy2);
1448 fiz2 = _mm_macc_pd(dz20,fscal,fiz2);
1450 fjx0 = _mm_macc_pd(dx20,fscal,fjx0);
1451 fjy0 = _mm_macc_pd(dy20,fscal,fjy0);
1452 fjz0 = _mm_macc_pd(dz20,fscal,fjz0);
1454 /**************************
1455 * CALCULATE INTERACTIONS *
1456 **************************/
1458 /* COULOMB ELECTROSTATICS */
1459 velec = _mm_mul_pd(qq21,rinv21);
1460 felec = _mm_mul_pd(velec,rinvsq21);
1464 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1466 /* Update vectorial force */
1467 fix2 = _mm_macc_pd(dx21,fscal,fix2);
1468 fiy2 = _mm_macc_pd(dy21,fscal,fiy2);
1469 fiz2 = _mm_macc_pd(dz21,fscal,fiz2);
1471 fjx1 = _mm_macc_pd(dx21,fscal,fjx1);
1472 fjy1 = _mm_macc_pd(dy21,fscal,fjy1);
1473 fjz1 = _mm_macc_pd(dz21,fscal,fjz1);
1475 /**************************
1476 * CALCULATE INTERACTIONS *
1477 **************************/
1479 /* COULOMB ELECTROSTATICS */
1480 velec = _mm_mul_pd(qq22,rinv22);
1481 felec = _mm_mul_pd(velec,rinvsq22);
1485 fscal = _mm_unpacklo_pd(fscal,_mm_setzero_pd());
1487 /* Update vectorial force */
1488 fix2 = _mm_macc_pd(dx22,fscal,fix2);
1489 fiy2 = _mm_macc_pd(dy22,fscal,fiy2);
1490 fiz2 = _mm_macc_pd(dz22,fscal,fiz2);
1492 fjx2 = _mm_macc_pd(dx22,fscal,fjx2);
1493 fjy2 = _mm_macc_pd(dy22,fscal,fjy2);
1494 fjz2 = _mm_macc_pd(dz22,fscal,fjz2);
1496 gmx_mm_decrement_3rvec_1ptr_swizzle_pd(f+j_coord_offsetA,fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1498 /* Inner loop uses 277 flops */
1501 /* End of innermost loop */
1503 gmx_mm_update_iforce_3atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1504 f+i_coord_offset,fshift+i_shift_offset);
1506 /* Increment number of inner iterations */
1507 inneriter += j_index_end - j_index_start;
1509 /* Outer loop uses 18 flops */
1512 /* Increment number of outer iterations */
1515 /* Update outer/inner flops */
1517 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*277);