2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
44 #include "../nb_kernel.h"
45 #include "types/simple.h"
49 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
50 #include "kernelutil_x86_avx_128_fma_single.h"
53 * Gromacs nonbonded kernel: nb_kernel_ElecEwSh_VdwLJSh_GeomW3W3_VF_avx_128_fma_single
54 * Electrostatics interaction: Ewald
55 * VdW interaction: LennardJones
56 * Geometry: Water3-Water3
57 * Calculate force/pot: PotentialAndForce
60 nb_kernel_ElecEwSh_VdwLJSh_GeomW3W3_VF_avx_128_fma_single
61 (t_nblist * gmx_restrict nlist,
62 rvec * gmx_restrict xx,
63 rvec * gmx_restrict ff,
64 t_forcerec * gmx_restrict fr,
65 t_mdatoms * gmx_restrict mdatoms,
66 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
67 t_nrnb * gmx_restrict nrnb)
69 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
70 * just 0 for non-waters.
71 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
72 * jnr indices corresponding to data put in the four positions in the SIMD register.
74 int i_shift_offset,i_coord_offset,outeriter,inneriter;
75 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
76 int jnrA,jnrB,jnrC,jnrD;
77 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
78 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
79 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
81 real *shiftvec,*fshift,*x,*f;
82 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
84 __m128 fscal,rcutoff,rcutoff2,jidxall;
86 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
88 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
90 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
91 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
92 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
93 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
94 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
95 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
96 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
97 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
98 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
99 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
100 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
101 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
102 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
103 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
104 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
105 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
106 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
109 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
112 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
113 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
115 __m128 ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
116 __m128 beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
118 __m128 dummy_mask,cutoff_mask;
119 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
120 __m128 one = _mm_set1_ps(1.0);
121 __m128 two = _mm_set1_ps(2.0);
127 jindex = nlist->jindex;
129 shiftidx = nlist->shift;
131 shiftvec = fr->shift_vec[0];
132 fshift = fr->fshift[0];
133 facel = _mm_set1_ps(fr->epsfac);
134 charge = mdatoms->chargeA;
135 nvdwtype = fr->ntype;
137 vdwtype = mdatoms->typeA;
139 sh_ewald = _mm_set1_ps(fr->ic->sh_ewald);
140 beta = _mm_set1_ps(fr->ic->ewaldcoeff);
141 beta2 = _mm_mul_ps(beta,beta);
142 beta3 = _mm_mul_ps(beta,beta2);
143 ewtab = fr->ic->tabq_coul_FDV0;
144 ewtabscale = _mm_set1_ps(fr->ic->tabq_scale);
145 ewtabhalfspace = _mm_set1_ps(0.5/fr->ic->tabq_scale);
147 /* Setup water-specific parameters */
148 inr = nlist->iinr[0];
149 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
150 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
151 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
152 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
154 jq0 = _mm_set1_ps(charge[inr+0]);
155 jq1 = _mm_set1_ps(charge[inr+1]);
156 jq2 = _mm_set1_ps(charge[inr+2]);
157 vdwjidx0A = 2*vdwtype[inr+0];
158 qq00 = _mm_mul_ps(iq0,jq0);
159 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
160 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
161 qq01 = _mm_mul_ps(iq0,jq1);
162 qq02 = _mm_mul_ps(iq0,jq2);
163 qq10 = _mm_mul_ps(iq1,jq0);
164 qq11 = _mm_mul_ps(iq1,jq1);
165 qq12 = _mm_mul_ps(iq1,jq2);
166 qq20 = _mm_mul_ps(iq2,jq0);
167 qq21 = _mm_mul_ps(iq2,jq1);
168 qq22 = _mm_mul_ps(iq2,jq2);
170 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
171 rcutoff_scalar = fr->rcoulomb;
172 rcutoff = _mm_set1_ps(rcutoff_scalar);
173 rcutoff2 = _mm_mul_ps(rcutoff,rcutoff);
175 sh_vdw_invrcut6 = _mm_set1_ps(fr->ic->sh_invrc6);
176 rvdw = _mm_set1_ps(fr->rvdw);
178 /* Avoid stupid compiler warnings */
179 jnrA = jnrB = jnrC = jnrD = 0;
188 for(iidx=0;iidx<4*DIM;iidx++)
193 /* Start outer loop over neighborlists */
194 for(iidx=0; iidx<nri; iidx++)
196 /* Load shift vector for this list */
197 i_shift_offset = DIM*shiftidx[iidx];
199 /* Load limits for loop over neighbors */
200 j_index_start = jindex[iidx];
201 j_index_end = jindex[iidx+1];
203 /* Get outer coordinate index */
205 i_coord_offset = DIM*inr;
207 /* Load i particle coords and add shift vector */
208 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
209 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
211 fix0 = _mm_setzero_ps();
212 fiy0 = _mm_setzero_ps();
213 fiz0 = _mm_setzero_ps();
214 fix1 = _mm_setzero_ps();
215 fiy1 = _mm_setzero_ps();
216 fiz1 = _mm_setzero_ps();
217 fix2 = _mm_setzero_ps();
218 fiy2 = _mm_setzero_ps();
219 fiz2 = _mm_setzero_ps();
221 /* Reset potential sums */
222 velecsum = _mm_setzero_ps();
223 vvdwsum = _mm_setzero_ps();
225 /* Start inner kernel loop */
226 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
229 /* Get j neighbor index, and coordinate index */
234 j_coord_offsetA = DIM*jnrA;
235 j_coord_offsetB = DIM*jnrB;
236 j_coord_offsetC = DIM*jnrC;
237 j_coord_offsetD = DIM*jnrD;
239 /* load j atom coordinates */
240 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
241 x+j_coord_offsetC,x+j_coord_offsetD,
242 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
244 /* Calculate displacement vector */
245 dx00 = _mm_sub_ps(ix0,jx0);
246 dy00 = _mm_sub_ps(iy0,jy0);
247 dz00 = _mm_sub_ps(iz0,jz0);
248 dx01 = _mm_sub_ps(ix0,jx1);
249 dy01 = _mm_sub_ps(iy0,jy1);
250 dz01 = _mm_sub_ps(iz0,jz1);
251 dx02 = _mm_sub_ps(ix0,jx2);
252 dy02 = _mm_sub_ps(iy0,jy2);
253 dz02 = _mm_sub_ps(iz0,jz2);
254 dx10 = _mm_sub_ps(ix1,jx0);
255 dy10 = _mm_sub_ps(iy1,jy0);
256 dz10 = _mm_sub_ps(iz1,jz0);
257 dx11 = _mm_sub_ps(ix1,jx1);
258 dy11 = _mm_sub_ps(iy1,jy1);
259 dz11 = _mm_sub_ps(iz1,jz1);
260 dx12 = _mm_sub_ps(ix1,jx2);
261 dy12 = _mm_sub_ps(iy1,jy2);
262 dz12 = _mm_sub_ps(iz1,jz2);
263 dx20 = _mm_sub_ps(ix2,jx0);
264 dy20 = _mm_sub_ps(iy2,jy0);
265 dz20 = _mm_sub_ps(iz2,jz0);
266 dx21 = _mm_sub_ps(ix2,jx1);
267 dy21 = _mm_sub_ps(iy2,jy1);
268 dz21 = _mm_sub_ps(iz2,jz1);
269 dx22 = _mm_sub_ps(ix2,jx2);
270 dy22 = _mm_sub_ps(iy2,jy2);
271 dz22 = _mm_sub_ps(iz2,jz2);
273 /* Calculate squared distance and things based on it */
274 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
275 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
276 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
277 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
278 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
279 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
280 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
281 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
282 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
284 rinv00 = gmx_mm_invsqrt_ps(rsq00);
285 rinv01 = gmx_mm_invsqrt_ps(rsq01);
286 rinv02 = gmx_mm_invsqrt_ps(rsq02);
287 rinv10 = gmx_mm_invsqrt_ps(rsq10);
288 rinv11 = gmx_mm_invsqrt_ps(rsq11);
289 rinv12 = gmx_mm_invsqrt_ps(rsq12);
290 rinv20 = gmx_mm_invsqrt_ps(rsq20);
291 rinv21 = gmx_mm_invsqrt_ps(rsq21);
292 rinv22 = gmx_mm_invsqrt_ps(rsq22);
294 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
295 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
296 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
297 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
298 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
299 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
300 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
301 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
302 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
304 fjx0 = _mm_setzero_ps();
305 fjy0 = _mm_setzero_ps();
306 fjz0 = _mm_setzero_ps();
307 fjx1 = _mm_setzero_ps();
308 fjy1 = _mm_setzero_ps();
309 fjz1 = _mm_setzero_ps();
310 fjx2 = _mm_setzero_ps();
311 fjy2 = _mm_setzero_ps();
312 fjz2 = _mm_setzero_ps();
314 /**************************
315 * CALCULATE INTERACTIONS *
316 **************************/
318 if (gmx_mm_any_lt(rsq00,rcutoff2))
321 r00 = _mm_mul_ps(rsq00,rinv00);
323 /* EWALD ELECTROSTATICS */
325 /* Analytical PME correction */
326 zeta2 = _mm_mul_ps(beta2,rsq00);
327 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
328 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
329 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
330 felec = _mm_mul_ps(qq00,felec);
331 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
332 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv00,sh_ewald));
333 velec = _mm_mul_ps(qq00,velec);
335 /* LENNARD-JONES DISPERSION/REPULSION */
337 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
338 vvdw6 = _mm_mul_ps(c6_00,rinvsix);
339 vvdw12 = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
340 vvdw = _mm_msub_ps(_mm_nmacc_ps(c12_00,_mm_mul_ps(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
341 _mm_mul_ps( _mm_nmacc_ps(c6_00,sh_vdw_invrcut6,vvdw6),one_sixth));
342 fvdw = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
344 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
346 /* Update potential sum for this i atom from the interaction with this j atom. */
347 velec = _mm_and_ps(velec,cutoff_mask);
348 velecsum = _mm_add_ps(velecsum,velec);
349 vvdw = _mm_and_ps(vvdw,cutoff_mask);
350 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
352 fscal = _mm_add_ps(felec,fvdw);
354 fscal = _mm_and_ps(fscal,cutoff_mask);
356 /* Update vectorial force */
357 fix0 = _mm_macc_ps(dx00,fscal,fix0);
358 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
359 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
361 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
362 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
363 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
367 /**************************
368 * CALCULATE INTERACTIONS *
369 **************************/
371 if (gmx_mm_any_lt(rsq01,rcutoff2))
374 r01 = _mm_mul_ps(rsq01,rinv01);
376 /* EWALD ELECTROSTATICS */
378 /* Analytical PME correction */
379 zeta2 = _mm_mul_ps(beta2,rsq01);
380 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
381 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
382 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
383 felec = _mm_mul_ps(qq01,felec);
384 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
385 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv01,sh_ewald));
386 velec = _mm_mul_ps(qq01,velec);
388 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
390 /* Update potential sum for this i atom from the interaction with this j atom. */
391 velec = _mm_and_ps(velec,cutoff_mask);
392 velecsum = _mm_add_ps(velecsum,velec);
396 fscal = _mm_and_ps(fscal,cutoff_mask);
398 /* Update vectorial force */
399 fix0 = _mm_macc_ps(dx01,fscal,fix0);
400 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
401 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
403 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
404 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
405 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
409 /**************************
410 * CALCULATE INTERACTIONS *
411 **************************/
413 if (gmx_mm_any_lt(rsq02,rcutoff2))
416 r02 = _mm_mul_ps(rsq02,rinv02);
418 /* EWALD ELECTROSTATICS */
420 /* Analytical PME correction */
421 zeta2 = _mm_mul_ps(beta2,rsq02);
422 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
423 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
424 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
425 felec = _mm_mul_ps(qq02,felec);
426 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
427 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv02,sh_ewald));
428 velec = _mm_mul_ps(qq02,velec);
430 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
432 /* Update potential sum for this i atom from the interaction with this j atom. */
433 velec = _mm_and_ps(velec,cutoff_mask);
434 velecsum = _mm_add_ps(velecsum,velec);
438 fscal = _mm_and_ps(fscal,cutoff_mask);
440 /* Update vectorial force */
441 fix0 = _mm_macc_ps(dx02,fscal,fix0);
442 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
443 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
445 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
446 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
447 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
451 /**************************
452 * CALCULATE INTERACTIONS *
453 **************************/
455 if (gmx_mm_any_lt(rsq10,rcutoff2))
458 r10 = _mm_mul_ps(rsq10,rinv10);
460 /* EWALD ELECTROSTATICS */
462 /* Analytical PME correction */
463 zeta2 = _mm_mul_ps(beta2,rsq10);
464 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
465 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
466 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
467 felec = _mm_mul_ps(qq10,felec);
468 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
469 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv10,sh_ewald));
470 velec = _mm_mul_ps(qq10,velec);
472 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
474 /* Update potential sum for this i atom from the interaction with this j atom. */
475 velec = _mm_and_ps(velec,cutoff_mask);
476 velecsum = _mm_add_ps(velecsum,velec);
480 fscal = _mm_and_ps(fscal,cutoff_mask);
482 /* Update vectorial force */
483 fix1 = _mm_macc_ps(dx10,fscal,fix1);
484 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
485 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
487 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
488 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
489 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
493 /**************************
494 * CALCULATE INTERACTIONS *
495 **************************/
497 if (gmx_mm_any_lt(rsq11,rcutoff2))
500 r11 = _mm_mul_ps(rsq11,rinv11);
502 /* EWALD ELECTROSTATICS */
504 /* Analytical PME correction */
505 zeta2 = _mm_mul_ps(beta2,rsq11);
506 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
507 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
508 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
509 felec = _mm_mul_ps(qq11,felec);
510 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
511 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv11,sh_ewald));
512 velec = _mm_mul_ps(qq11,velec);
514 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
516 /* Update potential sum for this i atom from the interaction with this j atom. */
517 velec = _mm_and_ps(velec,cutoff_mask);
518 velecsum = _mm_add_ps(velecsum,velec);
522 fscal = _mm_and_ps(fscal,cutoff_mask);
524 /* Update vectorial force */
525 fix1 = _mm_macc_ps(dx11,fscal,fix1);
526 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
527 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
529 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
530 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
531 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
535 /**************************
536 * CALCULATE INTERACTIONS *
537 **************************/
539 if (gmx_mm_any_lt(rsq12,rcutoff2))
542 r12 = _mm_mul_ps(rsq12,rinv12);
544 /* EWALD ELECTROSTATICS */
546 /* Analytical PME correction */
547 zeta2 = _mm_mul_ps(beta2,rsq12);
548 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
549 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
550 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
551 felec = _mm_mul_ps(qq12,felec);
552 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
553 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv12,sh_ewald));
554 velec = _mm_mul_ps(qq12,velec);
556 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
558 /* Update potential sum for this i atom from the interaction with this j atom. */
559 velec = _mm_and_ps(velec,cutoff_mask);
560 velecsum = _mm_add_ps(velecsum,velec);
564 fscal = _mm_and_ps(fscal,cutoff_mask);
566 /* Update vectorial force */
567 fix1 = _mm_macc_ps(dx12,fscal,fix1);
568 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
569 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
571 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
572 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
573 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
577 /**************************
578 * CALCULATE INTERACTIONS *
579 **************************/
581 if (gmx_mm_any_lt(rsq20,rcutoff2))
584 r20 = _mm_mul_ps(rsq20,rinv20);
586 /* EWALD ELECTROSTATICS */
588 /* Analytical PME correction */
589 zeta2 = _mm_mul_ps(beta2,rsq20);
590 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
591 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
592 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
593 felec = _mm_mul_ps(qq20,felec);
594 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
595 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv20,sh_ewald));
596 velec = _mm_mul_ps(qq20,velec);
598 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
600 /* Update potential sum for this i atom from the interaction with this j atom. */
601 velec = _mm_and_ps(velec,cutoff_mask);
602 velecsum = _mm_add_ps(velecsum,velec);
606 fscal = _mm_and_ps(fscal,cutoff_mask);
608 /* Update vectorial force */
609 fix2 = _mm_macc_ps(dx20,fscal,fix2);
610 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
611 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
613 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
614 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
615 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
619 /**************************
620 * CALCULATE INTERACTIONS *
621 **************************/
623 if (gmx_mm_any_lt(rsq21,rcutoff2))
626 r21 = _mm_mul_ps(rsq21,rinv21);
628 /* EWALD ELECTROSTATICS */
630 /* Analytical PME correction */
631 zeta2 = _mm_mul_ps(beta2,rsq21);
632 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
633 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
634 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
635 felec = _mm_mul_ps(qq21,felec);
636 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
637 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv21,sh_ewald));
638 velec = _mm_mul_ps(qq21,velec);
640 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
642 /* Update potential sum for this i atom from the interaction with this j atom. */
643 velec = _mm_and_ps(velec,cutoff_mask);
644 velecsum = _mm_add_ps(velecsum,velec);
648 fscal = _mm_and_ps(fscal,cutoff_mask);
650 /* Update vectorial force */
651 fix2 = _mm_macc_ps(dx21,fscal,fix2);
652 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
653 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
655 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
656 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
657 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
661 /**************************
662 * CALCULATE INTERACTIONS *
663 **************************/
665 if (gmx_mm_any_lt(rsq22,rcutoff2))
668 r22 = _mm_mul_ps(rsq22,rinv22);
670 /* EWALD ELECTROSTATICS */
672 /* Analytical PME correction */
673 zeta2 = _mm_mul_ps(beta2,rsq22);
674 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
675 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
676 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
677 felec = _mm_mul_ps(qq22,felec);
678 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
679 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv22,sh_ewald));
680 velec = _mm_mul_ps(qq22,velec);
682 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
684 /* Update potential sum for this i atom from the interaction with this j atom. */
685 velec = _mm_and_ps(velec,cutoff_mask);
686 velecsum = _mm_add_ps(velecsum,velec);
690 fscal = _mm_and_ps(fscal,cutoff_mask);
692 /* Update vectorial force */
693 fix2 = _mm_macc_ps(dx22,fscal,fix2);
694 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
695 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
697 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
698 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
699 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
703 fjptrA = f+j_coord_offsetA;
704 fjptrB = f+j_coord_offsetB;
705 fjptrC = f+j_coord_offsetC;
706 fjptrD = f+j_coord_offsetD;
708 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
709 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
711 /* Inner loop uses 315 flops */
717 /* Get j neighbor index, and coordinate index */
718 jnrlistA = jjnr[jidx];
719 jnrlistB = jjnr[jidx+1];
720 jnrlistC = jjnr[jidx+2];
721 jnrlistD = jjnr[jidx+3];
722 /* Sign of each element will be negative for non-real atoms.
723 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
724 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
726 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
727 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
728 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
729 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
730 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
731 j_coord_offsetA = DIM*jnrA;
732 j_coord_offsetB = DIM*jnrB;
733 j_coord_offsetC = DIM*jnrC;
734 j_coord_offsetD = DIM*jnrD;
736 /* load j atom coordinates */
737 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
738 x+j_coord_offsetC,x+j_coord_offsetD,
739 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
741 /* Calculate displacement vector */
742 dx00 = _mm_sub_ps(ix0,jx0);
743 dy00 = _mm_sub_ps(iy0,jy0);
744 dz00 = _mm_sub_ps(iz0,jz0);
745 dx01 = _mm_sub_ps(ix0,jx1);
746 dy01 = _mm_sub_ps(iy0,jy1);
747 dz01 = _mm_sub_ps(iz0,jz1);
748 dx02 = _mm_sub_ps(ix0,jx2);
749 dy02 = _mm_sub_ps(iy0,jy2);
750 dz02 = _mm_sub_ps(iz0,jz2);
751 dx10 = _mm_sub_ps(ix1,jx0);
752 dy10 = _mm_sub_ps(iy1,jy0);
753 dz10 = _mm_sub_ps(iz1,jz0);
754 dx11 = _mm_sub_ps(ix1,jx1);
755 dy11 = _mm_sub_ps(iy1,jy1);
756 dz11 = _mm_sub_ps(iz1,jz1);
757 dx12 = _mm_sub_ps(ix1,jx2);
758 dy12 = _mm_sub_ps(iy1,jy2);
759 dz12 = _mm_sub_ps(iz1,jz2);
760 dx20 = _mm_sub_ps(ix2,jx0);
761 dy20 = _mm_sub_ps(iy2,jy0);
762 dz20 = _mm_sub_ps(iz2,jz0);
763 dx21 = _mm_sub_ps(ix2,jx1);
764 dy21 = _mm_sub_ps(iy2,jy1);
765 dz21 = _mm_sub_ps(iz2,jz1);
766 dx22 = _mm_sub_ps(ix2,jx2);
767 dy22 = _mm_sub_ps(iy2,jy2);
768 dz22 = _mm_sub_ps(iz2,jz2);
770 /* Calculate squared distance and things based on it */
771 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
772 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
773 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
774 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
775 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
776 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
777 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
778 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
779 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
781 rinv00 = gmx_mm_invsqrt_ps(rsq00);
782 rinv01 = gmx_mm_invsqrt_ps(rsq01);
783 rinv02 = gmx_mm_invsqrt_ps(rsq02);
784 rinv10 = gmx_mm_invsqrt_ps(rsq10);
785 rinv11 = gmx_mm_invsqrt_ps(rsq11);
786 rinv12 = gmx_mm_invsqrt_ps(rsq12);
787 rinv20 = gmx_mm_invsqrt_ps(rsq20);
788 rinv21 = gmx_mm_invsqrt_ps(rsq21);
789 rinv22 = gmx_mm_invsqrt_ps(rsq22);
791 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
792 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
793 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
794 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
795 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
796 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
797 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
798 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
799 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
801 fjx0 = _mm_setzero_ps();
802 fjy0 = _mm_setzero_ps();
803 fjz0 = _mm_setzero_ps();
804 fjx1 = _mm_setzero_ps();
805 fjy1 = _mm_setzero_ps();
806 fjz1 = _mm_setzero_ps();
807 fjx2 = _mm_setzero_ps();
808 fjy2 = _mm_setzero_ps();
809 fjz2 = _mm_setzero_ps();
811 /**************************
812 * CALCULATE INTERACTIONS *
813 **************************/
815 if (gmx_mm_any_lt(rsq00,rcutoff2))
818 r00 = _mm_mul_ps(rsq00,rinv00);
819 r00 = _mm_andnot_ps(dummy_mask,r00);
821 /* EWALD ELECTROSTATICS */
823 /* Analytical PME correction */
824 zeta2 = _mm_mul_ps(beta2,rsq00);
825 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
826 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
827 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
828 felec = _mm_mul_ps(qq00,felec);
829 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
830 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv00,sh_ewald));
831 velec = _mm_mul_ps(qq00,velec);
833 /* LENNARD-JONES DISPERSION/REPULSION */
835 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
836 vvdw6 = _mm_mul_ps(c6_00,rinvsix);
837 vvdw12 = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
838 vvdw = _mm_msub_ps(_mm_nmacc_ps(c12_00,_mm_mul_ps(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
839 _mm_mul_ps( _mm_nmacc_ps(c6_00,sh_vdw_invrcut6,vvdw6),one_sixth));
840 fvdw = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
842 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
844 /* Update potential sum for this i atom from the interaction with this j atom. */
845 velec = _mm_and_ps(velec,cutoff_mask);
846 velec = _mm_andnot_ps(dummy_mask,velec);
847 velecsum = _mm_add_ps(velecsum,velec);
848 vvdw = _mm_and_ps(vvdw,cutoff_mask);
849 vvdw = _mm_andnot_ps(dummy_mask,vvdw);
850 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
852 fscal = _mm_add_ps(felec,fvdw);
854 fscal = _mm_and_ps(fscal,cutoff_mask);
856 fscal = _mm_andnot_ps(dummy_mask,fscal);
858 /* Update vectorial force */
859 fix0 = _mm_macc_ps(dx00,fscal,fix0);
860 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
861 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
863 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
864 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
865 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
869 /**************************
870 * CALCULATE INTERACTIONS *
871 **************************/
873 if (gmx_mm_any_lt(rsq01,rcutoff2))
876 r01 = _mm_mul_ps(rsq01,rinv01);
877 r01 = _mm_andnot_ps(dummy_mask,r01);
879 /* EWALD ELECTROSTATICS */
881 /* Analytical PME correction */
882 zeta2 = _mm_mul_ps(beta2,rsq01);
883 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
884 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
885 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
886 felec = _mm_mul_ps(qq01,felec);
887 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
888 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv01,sh_ewald));
889 velec = _mm_mul_ps(qq01,velec);
891 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
893 /* Update potential sum for this i atom from the interaction with this j atom. */
894 velec = _mm_and_ps(velec,cutoff_mask);
895 velec = _mm_andnot_ps(dummy_mask,velec);
896 velecsum = _mm_add_ps(velecsum,velec);
900 fscal = _mm_and_ps(fscal,cutoff_mask);
902 fscal = _mm_andnot_ps(dummy_mask,fscal);
904 /* Update vectorial force */
905 fix0 = _mm_macc_ps(dx01,fscal,fix0);
906 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
907 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
909 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
910 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
911 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
915 /**************************
916 * CALCULATE INTERACTIONS *
917 **************************/
919 if (gmx_mm_any_lt(rsq02,rcutoff2))
922 r02 = _mm_mul_ps(rsq02,rinv02);
923 r02 = _mm_andnot_ps(dummy_mask,r02);
925 /* EWALD ELECTROSTATICS */
927 /* Analytical PME correction */
928 zeta2 = _mm_mul_ps(beta2,rsq02);
929 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
930 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
931 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
932 felec = _mm_mul_ps(qq02,felec);
933 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
934 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv02,sh_ewald));
935 velec = _mm_mul_ps(qq02,velec);
937 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
939 /* Update potential sum for this i atom from the interaction with this j atom. */
940 velec = _mm_and_ps(velec,cutoff_mask);
941 velec = _mm_andnot_ps(dummy_mask,velec);
942 velecsum = _mm_add_ps(velecsum,velec);
946 fscal = _mm_and_ps(fscal,cutoff_mask);
948 fscal = _mm_andnot_ps(dummy_mask,fscal);
950 /* Update vectorial force */
951 fix0 = _mm_macc_ps(dx02,fscal,fix0);
952 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
953 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
955 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
956 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
957 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
961 /**************************
962 * CALCULATE INTERACTIONS *
963 **************************/
965 if (gmx_mm_any_lt(rsq10,rcutoff2))
968 r10 = _mm_mul_ps(rsq10,rinv10);
969 r10 = _mm_andnot_ps(dummy_mask,r10);
971 /* EWALD ELECTROSTATICS */
973 /* Analytical PME correction */
974 zeta2 = _mm_mul_ps(beta2,rsq10);
975 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
976 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
977 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
978 felec = _mm_mul_ps(qq10,felec);
979 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
980 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv10,sh_ewald));
981 velec = _mm_mul_ps(qq10,velec);
983 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
985 /* Update potential sum for this i atom from the interaction with this j atom. */
986 velec = _mm_and_ps(velec,cutoff_mask);
987 velec = _mm_andnot_ps(dummy_mask,velec);
988 velecsum = _mm_add_ps(velecsum,velec);
992 fscal = _mm_and_ps(fscal,cutoff_mask);
994 fscal = _mm_andnot_ps(dummy_mask,fscal);
996 /* Update vectorial force */
997 fix1 = _mm_macc_ps(dx10,fscal,fix1);
998 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
999 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1001 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1002 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1003 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1007 /**************************
1008 * CALCULATE INTERACTIONS *
1009 **************************/
1011 if (gmx_mm_any_lt(rsq11,rcutoff2))
1014 r11 = _mm_mul_ps(rsq11,rinv11);
1015 r11 = _mm_andnot_ps(dummy_mask,r11);
1017 /* EWALD ELECTROSTATICS */
1019 /* Analytical PME correction */
1020 zeta2 = _mm_mul_ps(beta2,rsq11);
1021 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
1022 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1023 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1024 felec = _mm_mul_ps(qq11,felec);
1025 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
1026 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv11,sh_ewald));
1027 velec = _mm_mul_ps(qq11,velec);
1029 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
1031 /* Update potential sum for this i atom from the interaction with this j atom. */
1032 velec = _mm_and_ps(velec,cutoff_mask);
1033 velec = _mm_andnot_ps(dummy_mask,velec);
1034 velecsum = _mm_add_ps(velecsum,velec);
1038 fscal = _mm_and_ps(fscal,cutoff_mask);
1040 fscal = _mm_andnot_ps(dummy_mask,fscal);
1042 /* Update vectorial force */
1043 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1044 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1045 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1047 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1048 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1049 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1053 /**************************
1054 * CALCULATE INTERACTIONS *
1055 **************************/
1057 if (gmx_mm_any_lt(rsq12,rcutoff2))
1060 r12 = _mm_mul_ps(rsq12,rinv12);
1061 r12 = _mm_andnot_ps(dummy_mask,r12);
1063 /* EWALD ELECTROSTATICS */
1065 /* Analytical PME correction */
1066 zeta2 = _mm_mul_ps(beta2,rsq12);
1067 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
1068 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1069 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1070 felec = _mm_mul_ps(qq12,felec);
1071 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
1072 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv12,sh_ewald));
1073 velec = _mm_mul_ps(qq12,velec);
1075 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
1077 /* Update potential sum for this i atom from the interaction with this j atom. */
1078 velec = _mm_and_ps(velec,cutoff_mask);
1079 velec = _mm_andnot_ps(dummy_mask,velec);
1080 velecsum = _mm_add_ps(velecsum,velec);
1084 fscal = _mm_and_ps(fscal,cutoff_mask);
1086 fscal = _mm_andnot_ps(dummy_mask,fscal);
1088 /* Update vectorial force */
1089 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1090 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1091 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1093 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1094 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1095 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1099 /**************************
1100 * CALCULATE INTERACTIONS *
1101 **************************/
1103 if (gmx_mm_any_lt(rsq20,rcutoff2))
1106 r20 = _mm_mul_ps(rsq20,rinv20);
1107 r20 = _mm_andnot_ps(dummy_mask,r20);
1109 /* EWALD ELECTROSTATICS */
1111 /* Analytical PME correction */
1112 zeta2 = _mm_mul_ps(beta2,rsq20);
1113 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
1114 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1115 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1116 felec = _mm_mul_ps(qq20,felec);
1117 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
1118 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv20,sh_ewald));
1119 velec = _mm_mul_ps(qq20,velec);
1121 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
1123 /* Update potential sum for this i atom from the interaction with this j atom. */
1124 velec = _mm_and_ps(velec,cutoff_mask);
1125 velec = _mm_andnot_ps(dummy_mask,velec);
1126 velecsum = _mm_add_ps(velecsum,velec);
1130 fscal = _mm_and_ps(fscal,cutoff_mask);
1132 fscal = _mm_andnot_ps(dummy_mask,fscal);
1134 /* Update vectorial force */
1135 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1136 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1137 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1139 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1140 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1141 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1145 /**************************
1146 * CALCULATE INTERACTIONS *
1147 **************************/
1149 if (gmx_mm_any_lt(rsq21,rcutoff2))
1152 r21 = _mm_mul_ps(rsq21,rinv21);
1153 r21 = _mm_andnot_ps(dummy_mask,r21);
1155 /* EWALD ELECTROSTATICS */
1157 /* Analytical PME correction */
1158 zeta2 = _mm_mul_ps(beta2,rsq21);
1159 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
1160 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1161 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1162 felec = _mm_mul_ps(qq21,felec);
1163 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
1164 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv21,sh_ewald));
1165 velec = _mm_mul_ps(qq21,velec);
1167 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
1169 /* Update potential sum for this i atom from the interaction with this j atom. */
1170 velec = _mm_and_ps(velec,cutoff_mask);
1171 velec = _mm_andnot_ps(dummy_mask,velec);
1172 velecsum = _mm_add_ps(velecsum,velec);
1176 fscal = _mm_and_ps(fscal,cutoff_mask);
1178 fscal = _mm_andnot_ps(dummy_mask,fscal);
1180 /* Update vectorial force */
1181 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1182 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1183 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1185 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1186 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1187 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1191 /**************************
1192 * CALCULATE INTERACTIONS *
1193 **************************/
1195 if (gmx_mm_any_lt(rsq22,rcutoff2))
1198 r22 = _mm_mul_ps(rsq22,rinv22);
1199 r22 = _mm_andnot_ps(dummy_mask,r22);
1201 /* EWALD ELECTROSTATICS */
1203 /* Analytical PME correction */
1204 zeta2 = _mm_mul_ps(beta2,rsq22);
1205 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
1206 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1207 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1208 felec = _mm_mul_ps(qq22,felec);
1209 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
1210 velec = _mm_nmacc_ps(pmecorrV,beta,_mm_sub_ps(rinv22,sh_ewald));
1211 velec = _mm_mul_ps(qq22,velec);
1213 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
1215 /* Update potential sum for this i atom from the interaction with this j atom. */
1216 velec = _mm_and_ps(velec,cutoff_mask);
1217 velec = _mm_andnot_ps(dummy_mask,velec);
1218 velecsum = _mm_add_ps(velecsum,velec);
1222 fscal = _mm_and_ps(fscal,cutoff_mask);
1224 fscal = _mm_andnot_ps(dummy_mask,fscal);
1226 /* Update vectorial force */
1227 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1228 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1229 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1231 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1232 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1233 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1237 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1238 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1239 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1240 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1242 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1243 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1245 /* Inner loop uses 324 flops */
1248 /* End of innermost loop */
1250 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1251 f+i_coord_offset,fshift+i_shift_offset);
1254 /* Update potential energies */
1255 gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1256 gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1258 /* Increment number of inner iterations */
1259 inneriter += j_index_end - j_index_start;
1261 /* Outer loop uses 20 flops */
1264 /* Increment number of outer iterations */
1267 /* Update outer/inner flops */
1269 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*324);
1272 * Gromacs nonbonded kernel: nb_kernel_ElecEwSh_VdwLJSh_GeomW3W3_F_avx_128_fma_single
1273 * Electrostatics interaction: Ewald
1274 * VdW interaction: LennardJones
1275 * Geometry: Water3-Water3
1276 * Calculate force/pot: Force
1279 nb_kernel_ElecEwSh_VdwLJSh_GeomW3W3_F_avx_128_fma_single
1280 (t_nblist * gmx_restrict nlist,
1281 rvec * gmx_restrict xx,
1282 rvec * gmx_restrict ff,
1283 t_forcerec * gmx_restrict fr,
1284 t_mdatoms * gmx_restrict mdatoms,
1285 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1286 t_nrnb * gmx_restrict nrnb)
1288 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1289 * just 0 for non-waters.
1290 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1291 * jnr indices corresponding to data put in the four positions in the SIMD register.
1293 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1294 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1295 int jnrA,jnrB,jnrC,jnrD;
1296 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1297 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1298 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1299 real rcutoff_scalar;
1300 real *shiftvec,*fshift,*x,*f;
1301 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1302 real scratch[4*DIM];
1303 __m128 fscal,rcutoff,rcutoff2,jidxall;
1305 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1307 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1309 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1310 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1311 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1312 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1313 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1314 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1315 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1316 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1317 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1318 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1319 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1320 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1321 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1322 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1323 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1324 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1325 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
1328 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1331 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
1332 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
1334 __m128 ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
1335 __m128 beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
1337 __m128 dummy_mask,cutoff_mask;
1338 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1339 __m128 one = _mm_set1_ps(1.0);
1340 __m128 two = _mm_set1_ps(2.0);
1346 jindex = nlist->jindex;
1348 shiftidx = nlist->shift;
1350 shiftvec = fr->shift_vec[0];
1351 fshift = fr->fshift[0];
1352 facel = _mm_set1_ps(fr->epsfac);
1353 charge = mdatoms->chargeA;
1354 nvdwtype = fr->ntype;
1355 vdwparam = fr->nbfp;
1356 vdwtype = mdatoms->typeA;
1358 sh_ewald = _mm_set1_ps(fr->ic->sh_ewald);
1359 beta = _mm_set1_ps(fr->ic->ewaldcoeff);
1360 beta2 = _mm_mul_ps(beta,beta);
1361 beta3 = _mm_mul_ps(beta,beta2);
1362 ewtab = fr->ic->tabq_coul_F;
1363 ewtabscale = _mm_set1_ps(fr->ic->tabq_scale);
1364 ewtabhalfspace = _mm_set1_ps(0.5/fr->ic->tabq_scale);
1366 /* Setup water-specific parameters */
1367 inr = nlist->iinr[0];
1368 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1369 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1370 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1371 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
1373 jq0 = _mm_set1_ps(charge[inr+0]);
1374 jq1 = _mm_set1_ps(charge[inr+1]);
1375 jq2 = _mm_set1_ps(charge[inr+2]);
1376 vdwjidx0A = 2*vdwtype[inr+0];
1377 qq00 = _mm_mul_ps(iq0,jq0);
1378 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1379 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1380 qq01 = _mm_mul_ps(iq0,jq1);
1381 qq02 = _mm_mul_ps(iq0,jq2);
1382 qq10 = _mm_mul_ps(iq1,jq0);
1383 qq11 = _mm_mul_ps(iq1,jq1);
1384 qq12 = _mm_mul_ps(iq1,jq2);
1385 qq20 = _mm_mul_ps(iq2,jq0);
1386 qq21 = _mm_mul_ps(iq2,jq1);
1387 qq22 = _mm_mul_ps(iq2,jq2);
1389 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
1390 rcutoff_scalar = fr->rcoulomb;
1391 rcutoff = _mm_set1_ps(rcutoff_scalar);
1392 rcutoff2 = _mm_mul_ps(rcutoff,rcutoff);
1394 sh_vdw_invrcut6 = _mm_set1_ps(fr->ic->sh_invrc6);
1395 rvdw = _mm_set1_ps(fr->rvdw);
1397 /* Avoid stupid compiler warnings */
1398 jnrA = jnrB = jnrC = jnrD = 0;
1399 j_coord_offsetA = 0;
1400 j_coord_offsetB = 0;
1401 j_coord_offsetC = 0;
1402 j_coord_offsetD = 0;
1407 for(iidx=0;iidx<4*DIM;iidx++)
1409 scratch[iidx] = 0.0;
1412 /* Start outer loop over neighborlists */
1413 for(iidx=0; iidx<nri; iidx++)
1415 /* Load shift vector for this list */
1416 i_shift_offset = DIM*shiftidx[iidx];
1418 /* Load limits for loop over neighbors */
1419 j_index_start = jindex[iidx];
1420 j_index_end = jindex[iidx+1];
1422 /* Get outer coordinate index */
1424 i_coord_offset = DIM*inr;
1426 /* Load i particle coords and add shift vector */
1427 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1428 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1430 fix0 = _mm_setzero_ps();
1431 fiy0 = _mm_setzero_ps();
1432 fiz0 = _mm_setzero_ps();
1433 fix1 = _mm_setzero_ps();
1434 fiy1 = _mm_setzero_ps();
1435 fiz1 = _mm_setzero_ps();
1436 fix2 = _mm_setzero_ps();
1437 fiy2 = _mm_setzero_ps();
1438 fiz2 = _mm_setzero_ps();
1440 /* Start inner kernel loop */
1441 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1444 /* Get j neighbor index, and coordinate index */
1446 jnrB = jjnr[jidx+1];
1447 jnrC = jjnr[jidx+2];
1448 jnrD = jjnr[jidx+3];
1449 j_coord_offsetA = DIM*jnrA;
1450 j_coord_offsetB = DIM*jnrB;
1451 j_coord_offsetC = DIM*jnrC;
1452 j_coord_offsetD = DIM*jnrD;
1454 /* load j atom coordinates */
1455 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1456 x+j_coord_offsetC,x+j_coord_offsetD,
1457 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1459 /* Calculate displacement vector */
1460 dx00 = _mm_sub_ps(ix0,jx0);
1461 dy00 = _mm_sub_ps(iy0,jy0);
1462 dz00 = _mm_sub_ps(iz0,jz0);
1463 dx01 = _mm_sub_ps(ix0,jx1);
1464 dy01 = _mm_sub_ps(iy0,jy1);
1465 dz01 = _mm_sub_ps(iz0,jz1);
1466 dx02 = _mm_sub_ps(ix0,jx2);
1467 dy02 = _mm_sub_ps(iy0,jy2);
1468 dz02 = _mm_sub_ps(iz0,jz2);
1469 dx10 = _mm_sub_ps(ix1,jx0);
1470 dy10 = _mm_sub_ps(iy1,jy0);
1471 dz10 = _mm_sub_ps(iz1,jz0);
1472 dx11 = _mm_sub_ps(ix1,jx1);
1473 dy11 = _mm_sub_ps(iy1,jy1);
1474 dz11 = _mm_sub_ps(iz1,jz1);
1475 dx12 = _mm_sub_ps(ix1,jx2);
1476 dy12 = _mm_sub_ps(iy1,jy2);
1477 dz12 = _mm_sub_ps(iz1,jz2);
1478 dx20 = _mm_sub_ps(ix2,jx0);
1479 dy20 = _mm_sub_ps(iy2,jy0);
1480 dz20 = _mm_sub_ps(iz2,jz0);
1481 dx21 = _mm_sub_ps(ix2,jx1);
1482 dy21 = _mm_sub_ps(iy2,jy1);
1483 dz21 = _mm_sub_ps(iz2,jz1);
1484 dx22 = _mm_sub_ps(ix2,jx2);
1485 dy22 = _mm_sub_ps(iy2,jy2);
1486 dz22 = _mm_sub_ps(iz2,jz2);
1488 /* Calculate squared distance and things based on it */
1489 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1490 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1491 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1492 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1493 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1494 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1495 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1496 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1497 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1499 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1500 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1501 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1502 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1503 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1504 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1505 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1506 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1507 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1509 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1510 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1511 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1512 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1513 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1514 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1515 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1516 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1517 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1519 fjx0 = _mm_setzero_ps();
1520 fjy0 = _mm_setzero_ps();
1521 fjz0 = _mm_setzero_ps();
1522 fjx1 = _mm_setzero_ps();
1523 fjy1 = _mm_setzero_ps();
1524 fjz1 = _mm_setzero_ps();
1525 fjx2 = _mm_setzero_ps();
1526 fjy2 = _mm_setzero_ps();
1527 fjz2 = _mm_setzero_ps();
1529 /**************************
1530 * CALCULATE INTERACTIONS *
1531 **************************/
1533 if (gmx_mm_any_lt(rsq00,rcutoff2))
1536 r00 = _mm_mul_ps(rsq00,rinv00);
1538 /* EWALD ELECTROSTATICS */
1540 /* Analytical PME correction */
1541 zeta2 = _mm_mul_ps(beta2,rsq00);
1542 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
1543 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1544 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1545 felec = _mm_mul_ps(qq00,felec);
1547 /* LENNARD-JONES DISPERSION/REPULSION */
1549 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1550 fvdw = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1552 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
1554 fscal = _mm_add_ps(felec,fvdw);
1556 fscal = _mm_and_ps(fscal,cutoff_mask);
1558 /* Update vectorial force */
1559 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1560 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1561 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1563 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1564 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1565 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1569 /**************************
1570 * CALCULATE INTERACTIONS *
1571 **************************/
1573 if (gmx_mm_any_lt(rsq01,rcutoff2))
1576 r01 = _mm_mul_ps(rsq01,rinv01);
1578 /* EWALD ELECTROSTATICS */
1580 /* Analytical PME correction */
1581 zeta2 = _mm_mul_ps(beta2,rsq01);
1582 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
1583 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1584 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1585 felec = _mm_mul_ps(qq01,felec);
1587 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
1591 fscal = _mm_and_ps(fscal,cutoff_mask);
1593 /* Update vectorial force */
1594 fix0 = _mm_macc_ps(dx01,fscal,fix0);
1595 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
1596 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
1598 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
1599 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
1600 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
1604 /**************************
1605 * CALCULATE INTERACTIONS *
1606 **************************/
1608 if (gmx_mm_any_lt(rsq02,rcutoff2))
1611 r02 = _mm_mul_ps(rsq02,rinv02);
1613 /* EWALD ELECTROSTATICS */
1615 /* Analytical PME correction */
1616 zeta2 = _mm_mul_ps(beta2,rsq02);
1617 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
1618 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1619 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1620 felec = _mm_mul_ps(qq02,felec);
1622 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
1626 fscal = _mm_and_ps(fscal,cutoff_mask);
1628 /* Update vectorial force */
1629 fix0 = _mm_macc_ps(dx02,fscal,fix0);
1630 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
1631 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
1633 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
1634 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
1635 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
1639 /**************************
1640 * CALCULATE INTERACTIONS *
1641 **************************/
1643 if (gmx_mm_any_lt(rsq10,rcutoff2))
1646 r10 = _mm_mul_ps(rsq10,rinv10);
1648 /* EWALD ELECTROSTATICS */
1650 /* Analytical PME correction */
1651 zeta2 = _mm_mul_ps(beta2,rsq10);
1652 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
1653 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1654 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1655 felec = _mm_mul_ps(qq10,felec);
1657 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
1661 fscal = _mm_and_ps(fscal,cutoff_mask);
1663 /* Update vectorial force */
1664 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1665 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1666 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1668 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1669 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1670 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1674 /**************************
1675 * CALCULATE INTERACTIONS *
1676 **************************/
1678 if (gmx_mm_any_lt(rsq11,rcutoff2))
1681 r11 = _mm_mul_ps(rsq11,rinv11);
1683 /* EWALD ELECTROSTATICS */
1685 /* Analytical PME correction */
1686 zeta2 = _mm_mul_ps(beta2,rsq11);
1687 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
1688 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1689 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1690 felec = _mm_mul_ps(qq11,felec);
1692 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
1696 fscal = _mm_and_ps(fscal,cutoff_mask);
1698 /* Update vectorial force */
1699 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1700 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1701 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1703 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1704 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1705 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1709 /**************************
1710 * CALCULATE INTERACTIONS *
1711 **************************/
1713 if (gmx_mm_any_lt(rsq12,rcutoff2))
1716 r12 = _mm_mul_ps(rsq12,rinv12);
1718 /* EWALD ELECTROSTATICS */
1720 /* Analytical PME correction */
1721 zeta2 = _mm_mul_ps(beta2,rsq12);
1722 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
1723 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1724 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1725 felec = _mm_mul_ps(qq12,felec);
1727 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
1731 fscal = _mm_and_ps(fscal,cutoff_mask);
1733 /* Update vectorial force */
1734 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1735 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1736 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1738 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1739 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1740 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1744 /**************************
1745 * CALCULATE INTERACTIONS *
1746 **************************/
1748 if (gmx_mm_any_lt(rsq20,rcutoff2))
1751 r20 = _mm_mul_ps(rsq20,rinv20);
1753 /* EWALD ELECTROSTATICS */
1755 /* Analytical PME correction */
1756 zeta2 = _mm_mul_ps(beta2,rsq20);
1757 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
1758 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1759 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1760 felec = _mm_mul_ps(qq20,felec);
1762 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
1766 fscal = _mm_and_ps(fscal,cutoff_mask);
1768 /* Update vectorial force */
1769 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1770 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1771 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1773 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1774 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1775 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1779 /**************************
1780 * CALCULATE INTERACTIONS *
1781 **************************/
1783 if (gmx_mm_any_lt(rsq21,rcutoff2))
1786 r21 = _mm_mul_ps(rsq21,rinv21);
1788 /* EWALD ELECTROSTATICS */
1790 /* Analytical PME correction */
1791 zeta2 = _mm_mul_ps(beta2,rsq21);
1792 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
1793 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1794 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1795 felec = _mm_mul_ps(qq21,felec);
1797 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
1801 fscal = _mm_and_ps(fscal,cutoff_mask);
1803 /* Update vectorial force */
1804 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1805 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1806 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1808 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1809 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1810 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1814 /**************************
1815 * CALCULATE INTERACTIONS *
1816 **************************/
1818 if (gmx_mm_any_lt(rsq22,rcutoff2))
1821 r22 = _mm_mul_ps(rsq22,rinv22);
1823 /* EWALD ELECTROSTATICS */
1825 /* Analytical PME correction */
1826 zeta2 = _mm_mul_ps(beta2,rsq22);
1827 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
1828 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1829 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1830 felec = _mm_mul_ps(qq22,felec);
1832 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
1836 fscal = _mm_and_ps(fscal,cutoff_mask);
1838 /* Update vectorial force */
1839 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1840 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1841 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1843 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1844 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1845 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1849 fjptrA = f+j_coord_offsetA;
1850 fjptrB = f+j_coord_offsetB;
1851 fjptrC = f+j_coord_offsetC;
1852 fjptrD = f+j_coord_offsetD;
1854 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1855 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1857 /* Inner loop uses 286 flops */
1860 if(jidx<j_index_end)
1863 /* Get j neighbor index, and coordinate index */
1864 jnrlistA = jjnr[jidx];
1865 jnrlistB = jjnr[jidx+1];
1866 jnrlistC = jjnr[jidx+2];
1867 jnrlistD = jjnr[jidx+3];
1868 /* Sign of each element will be negative for non-real atoms.
1869 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1870 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1872 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1873 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1874 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1875 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1876 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1877 j_coord_offsetA = DIM*jnrA;
1878 j_coord_offsetB = DIM*jnrB;
1879 j_coord_offsetC = DIM*jnrC;
1880 j_coord_offsetD = DIM*jnrD;
1882 /* load j atom coordinates */
1883 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1884 x+j_coord_offsetC,x+j_coord_offsetD,
1885 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1887 /* Calculate displacement vector */
1888 dx00 = _mm_sub_ps(ix0,jx0);
1889 dy00 = _mm_sub_ps(iy0,jy0);
1890 dz00 = _mm_sub_ps(iz0,jz0);
1891 dx01 = _mm_sub_ps(ix0,jx1);
1892 dy01 = _mm_sub_ps(iy0,jy1);
1893 dz01 = _mm_sub_ps(iz0,jz1);
1894 dx02 = _mm_sub_ps(ix0,jx2);
1895 dy02 = _mm_sub_ps(iy0,jy2);
1896 dz02 = _mm_sub_ps(iz0,jz2);
1897 dx10 = _mm_sub_ps(ix1,jx0);
1898 dy10 = _mm_sub_ps(iy1,jy0);
1899 dz10 = _mm_sub_ps(iz1,jz0);
1900 dx11 = _mm_sub_ps(ix1,jx1);
1901 dy11 = _mm_sub_ps(iy1,jy1);
1902 dz11 = _mm_sub_ps(iz1,jz1);
1903 dx12 = _mm_sub_ps(ix1,jx2);
1904 dy12 = _mm_sub_ps(iy1,jy2);
1905 dz12 = _mm_sub_ps(iz1,jz2);
1906 dx20 = _mm_sub_ps(ix2,jx0);
1907 dy20 = _mm_sub_ps(iy2,jy0);
1908 dz20 = _mm_sub_ps(iz2,jz0);
1909 dx21 = _mm_sub_ps(ix2,jx1);
1910 dy21 = _mm_sub_ps(iy2,jy1);
1911 dz21 = _mm_sub_ps(iz2,jz1);
1912 dx22 = _mm_sub_ps(ix2,jx2);
1913 dy22 = _mm_sub_ps(iy2,jy2);
1914 dz22 = _mm_sub_ps(iz2,jz2);
1916 /* Calculate squared distance and things based on it */
1917 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1918 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1919 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1920 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1921 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1922 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1923 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1924 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1925 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1927 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1928 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1929 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1930 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1931 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1932 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1933 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1934 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1935 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1937 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1938 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1939 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1940 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1941 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1942 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1943 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1944 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1945 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1947 fjx0 = _mm_setzero_ps();
1948 fjy0 = _mm_setzero_ps();
1949 fjz0 = _mm_setzero_ps();
1950 fjx1 = _mm_setzero_ps();
1951 fjy1 = _mm_setzero_ps();
1952 fjz1 = _mm_setzero_ps();
1953 fjx2 = _mm_setzero_ps();
1954 fjy2 = _mm_setzero_ps();
1955 fjz2 = _mm_setzero_ps();
1957 /**************************
1958 * CALCULATE INTERACTIONS *
1959 **************************/
1961 if (gmx_mm_any_lt(rsq00,rcutoff2))
1964 r00 = _mm_mul_ps(rsq00,rinv00);
1965 r00 = _mm_andnot_ps(dummy_mask,r00);
1967 /* EWALD ELECTROSTATICS */
1969 /* Analytical PME correction */
1970 zeta2 = _mm_mul_ps(beta2,rsq00);
1971 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
1972 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1973 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1974 felec = _mm_mul_ps(qq00,felec);
1976 /* LENNARD-JONES DISPERSION/REPULSION */
1978 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1979 fvdw = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1981 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
1983 fscal = _mm_add_ps(felec,fvdw);
1985 fscal = _mm_and_ps(fscal,cutoff_mask);
1987 fscal = _mm_andnot_ps(dummy_mask,fscal);
1989 /* Update vectorial force */
1990 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1991 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1992 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1994 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1995 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1996 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
2000 /**************************
2001 * CALCULATE INTERACTIONS *
2002 **************************/
2004 if (gmx_mm_any_lt(rsq01,rcutoff2))
2007 r01 = _mm_mul_ps(rsq01,rinv01);
2008 r01 = _mm_andnot_ps(dummy_mask,r01);
2010 /* EWALD ELECTROSTATICS */
2012 /* Analytical PME correction */
2013 zeta2 = _mm_mul_ps(beta2,rsq01);
2014 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
2015 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2016 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2017 felec = _mm_mul_ps(qq01,felec);
2019 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
2023 fscal = _mm_and_ps(fscal,cutoff_mask);
2025 fscal = _mm_andnot_ps(dummy_mask,fscal);
2027 /* Update vectorial force */
2028 fix0 = _mm_macc_ps(dx01,fscal,fix0);
2029 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
2030 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
2032 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
2033 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
2034 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
2038 /**************************
2039 * CALCULATE INTERACTIONS *
2040 **************************/
2042 if (gmx_mm_any_lt(rsq02,rcutoff2))
2045 r02 = _mm_mul_ps(rsq02,rinv02);
2046 r02 = _mm_andnot_ps(dummy_mask,r02);
2048 /* EWALD ELECTROSTATICS */
2050 /* Analytical PME correction */
2051 zeta2 = _mm_mul_ps(beta2,rsq02);
2052 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
2053 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2054 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2055 felec = _mm_mul_ps(qq02,felec);
2057 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
2061 fscal = _mm_and_ps(fscal,cutoff_mask);
2063 fscal = _mm_andnot_ps(dummy_mask,fscal);
2065 /* Update vectorial force */
2066 fix0 = _mm_macc_ps(dx02,fscal,fix0);
2067 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
2068 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
2070 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
2071 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
2072 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
2076 /**************************
2077 * CALCULATE INTERACTIONS *
2078 **************************/
2080 if (gmx_mm_any_lt(rsq10,rcutoff2))
2083 r10 = _mm_mul_ps(rsq10,rinv10);
2084 r10 = _mm_andnot_ps(dummy_mask,r10);
2086 /* EWALD ELECTROSTATICS */
2088 /* Analytical PME correction */
2089 zeta2 = _mm_mul_ps(beta2,rsq10);
2090 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
2091 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2092 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2093 felec = _mm_mul_ps(qq10,felec);
2095 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
2099 fscal = _mm_and_ps(fscal,cutoff_mask);
2101 fscal = _mm_andnot_ps(dummy_mask,fscal);
2103 /* Update vectorial force */
2104 fix1 = _mm_macc_ps(dx10,fscal,fix1);
2105 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
2106 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
2108 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
2109 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
2110 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
2114 /**************************
2115 * CALCULATE INTERACTIONS *
2116 **************************/
2118 if (gmx_mm_any_lt(rsq11,rcutoff2))
2121 r11 = _mm_mul_ps(rsq11,rinv11);
2122 r11 = _mm_andnot_ps(dummy_mask,r11);
2124 /* EWALD ELECTROSTATICS */
2126 /* Analytical PME correction */
2127 zeta2 = _mm_mul_ps(beta2,rsq11);
2128 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
2129 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2130 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2131 felec = _mm_mul_ps(qq11,felec);
2133 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
2137 fscal = _mm_and_ps(fscal,cutoff_mask);
2139 fscal = _mm_andnot_ps(dummy_mask,fscal);
2141 /* Update vectorial force */
2142 fix1 = _mm_macc_ps(dx11,fscal,fix1);
2143 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
2144 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
2146 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
2147 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
2148 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
2152 /**************************
2153 * CALCULATE INTERACTIONS *
2154 **************************/
2156 if (gmx_mm_any_lt(rsq12,rcutoff2))
2159 r12 = _mm_mul_ps(rsq12,rinv12);
2160 r12 = _mm_andnot_ps(dummy_mask,r12);
2162 /* EWALD ELECTROSTATICS */
2164 /* Analytical PME correction */
2165 zeta2 = _mm_mul_ps(beta2,rsq12);
2166 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
2167 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2168 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2169 felec = _mm_mul_ps(qq12,felec);
2171 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
2175 fscal = _mm_and_ps(fscal,cutoff_mask);
2177 fscal = _mm_andnot_ps(dummy_mask,fscal);
2179 /* Update vectorial force */
2180 fix1 = _mm_macc_ps(dx12,fscal,fix1);
2181 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
2182 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
2184 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
2185 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
2186 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
2190 /**************************
2191 * CALCULATE INTERACTIONS *
2192 **************************/
2194 if (gmx_mm_any_lt(rsq20,rcutoff2))
2197 r20 = _mm_mul_ps(rsq20,rinv20);
2198 r20 = _mm_andnot_ps(dummy_mask,r20);
2200 /* EWALD ELECTROSTATICS */
2202 /* Analytical PME correction */
2203 zeta2 = _mm_mul_ps(beta2,rsq20);
2204 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
2205 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2206 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2207 felec = _mm_mul_ps(qq20,felec);
2209 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
2213 fscal = _mm_and_ps(fscal,cutoff_mask);
2215 fscal = _mm_andnot_ps(dummy_mask,fscal);
2217 /* Update vectorial force */
2218 fix2 = _mm_macc_ps(dx20,fscal,fix2);
2219 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
2220 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
2222 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
2223 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
2224 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
2228 /**************************
2229 * CALCULATE INTERACTIONS *
2230 **************************/
2232 if (gmx_mm_any_lt(rsq21,rcutoff2))
2235 r21 = _mm_mul_ps(rsq21,rinv21);
2236 r21 = _mm_andnot_ps(dummy_mask,r21);
2238 /* EWALD ELECTROSTATICS */
2240 /* Analytical PME correction */
2241 zeta2 = _mm_mul_ps(beta2,rsq21);
2242 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
2243 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2244 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2245 felec = _mm_mul_ps(qq21,felec);
2247 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
2251 fscal = _mm_and_ps(fscal,cutoff_mask);
2253 fscal = _mm_andnot_ps(dummy_mask,fscal);
2255 /* Update vectorial force */
2256 fix2 = _mm_macc_ps(dx21,fscal,fix2);
2257 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
2258 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
2260 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
2261 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
2262 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
2266 /**************************
2267 * CALCULATE INTERACTIONS *
2268 **************************/
2270 if (gmx_mm_any_lt(rsq22,rcutoff2))
2273 r22 = _mm_mul_ps(rsq22,rinv22);
2274 r22 = _mm_andnot_ps(dummy_mask,r22);
2276 /* EWALD ELECTROSTATICS */
2278 /* Analytical PME correction */
2279 zeta2 = _mm_mul_ps(beta2,rsq22);
2280 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
2281 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
2282 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
2283 felec = _mm_mul_ps(qq22,felec);
2285 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
2289 fscal = _mm_and_ps(fscal,cutoff_mask);
2291 fscal = _mm_andnot_ps(dummy_mask,fscal);
2293 /* Update vectorial force */
2294 fix2 = _mm_macc_ps(dx22,fscal,fix2);
2295 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
2296 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
2298 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
2299 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
2300 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
2304 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
2305 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
2306 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
2307 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
2309 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
2310 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
2312 /* Inner loop uses 295 flops */
2315 /* End of innermost loop */
2317 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
2318 f+i_coord_offset,fshift+i_shift_offset);
2320 /* Increment number of inner iterations */
2321 inneriter += j_index_end - j_index_start;
2323 /* Outer loop uses 18 flops */
2326 /* Increment number of outer iterations */
2329 /* Update outer/inner flops */
2331 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*295);