2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
42 #include "../nb_kernel.h"
43 #include "gromacs/legacyheaders/types/simple.h"
44 #include "gromacs/math/vec.h"
45 #include "gromacs/legacyheaders/nrnb.h"
47 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
48 #include "kernelutil_x86_avx_128_fma_single.h"
51 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwLJSh_GeomW3W3_VF_avx_128_fma_single
52 * Electrostatics interaction: ReactionField
53 * VdW interaction: LennardJones
54 * Geometry: Water3-Water3
55 * Calculate force/pot: PotentialAndForce
58 nb_kernel_ElecRFCut_VdwLJSh_GeomW3W3_VF_avx_128_fma_single
59 (t_nblist * gmx_restrict nlist,
60 rvec * gmx_restrict xx,
61 rvec * gmx_restrict ff,
62 t_forcerec * gmx_restrict fr,
63 t_mdatoms * gmx_restrict mdatoms,
64 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
65 t_nrnb * gmx_restrict nrnb)
67 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
68 * just 0 for non-waters.
69 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
70 * jnr indices corresponding to data put in the four positions in the SIMD register.
72 int i_shift_offset,i_coord_offset,outeriter,inneriter;
73 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
74 int jnrA,jnrB,jnrC,jnrD;
75 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
76 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
77 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
79 real *shiftvec,*fshift,*x,*f;
80 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
82 __m128 fscal,rcutoff,rcutoff2,jidxall;
84 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
86 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
88 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
89 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
90 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
91 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
92 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
93 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
94 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
95 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
96 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
97 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
98 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
99 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
100 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
101 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
102 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
103 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
104 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
107 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
110 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
111 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
112 __m128 dummy_mask,cutoff_mask;
113 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
114 __m128 one = _mm_set1_ps(1.0);
115 __m128 two = _mm_set1_ps(2.0);
121 jindex = nlist->jindex;
123 shiftidx = nlist->shift;
125 shiftvec = fr->shift_vec[0];
126 fshift = fr->fshift[0];
127 facel = _mm_set1_ps(fr->epsfac);
128 charge = mdatoms->chargeA;
129 krf = _mm_set1_ps(fr->ic->k_rf);
130 krf2 = _mm_set1_ps(fr->ic->k_rf*2.0);
131 crf = _mm_set1_ps(fr->ic->c_rf);
132 nvdwtype = fr->ntype;
134 vdwtype = mdatoms->typeA;
136 /* Setup water-specific parameters */
137 inr = nlist->iinr[0];
138 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
139 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
140 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
141 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
143 jq0 = _mm_set1_ps(charge[inr+0]);
144 jq1 = _mm_set1_ps(charge[inr+1]);
145 jq2 = _mm_set1_ps(charge[inr+2]);
146 vdwjidx0A = 2*vdwtype[inr+0];
147 qq00 = _mm_mul_ps(iq0,jq0);
148 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
149 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
150 qq01 = _mm_mul_ps(iq0,jq1);
151 qq02 = _mm_mul_ps(iq0,jq2);
152 qq10 = _mm_mul_ps(iq1,jq0);
153 qq11 = _mm_mul_ps(iq1,jq1);
154 qq12 = _mm_mul_ps(iq1,jq2);
155 qq20 = _mm_mul_ps(iq2,jq0);
156 qq21 = _mm_mul_ps(iq2,jq1);
157 qq22 = _mm_mul_ps(iq2,jq2);
159 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
160 rcutoff_scalar = fr->rcoulomb;
161 rcutoff = _mm_set1_ps(rcutoff_scalar);
162 rcutoff2 = _mm_mul_ps(rcutoff,rcutoff);
164 sh_vdw_invrcut6 = _mm_set1_ps(fr->ic->sh_invrc6);
165 rvdw = _mm_set1_ps(fr->rvdw);
167 /* Avoid stupid compiler warnings */
168 jnrA = jnrB = jnrC = jnrD = 0;
177 for(iidx=0;iidx<4*DIM;iidx++)
182 /* Start outer loop over neighborlists */
183 for(iidx=0; iidx<nri; iidx++)
185 /* Load shift vector for this list */
186 i_shift_offset = DIM*shiftidx[iidx];
188 /* Load limits for loop over neighbors */
189 j_index_start = jindex[iidx];
190 j_index_end = jindex[iidx+1];
192 /* Get outer coordinate index */
194 i_coord_offset = DIM*inr;
196 /* Load i particle coords and add shift vector */
197 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
198 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
200 fix0 = _mm_setzero_ps();
201 fiy0 = _mm_setzero_ps();
202 fiz0 = _mm_setzero_ps();
203 fix1 = _mm_setzero_ps();
204 fiy1 = _mm_setzero_ps();
205 fiz1 = _mm_setzero_ps();
206 fix2 = _mm_setzero_ps();
207 fiy2 = _mm_setzero_ps();
208 fiz2 = _mm_setzero_ps();
210 /* Reset potential sums */
211 velecsum = _mm_setzero_ps();
212 vvdwsum = _mm_setzero_ps();
214 /* Start inner kernel loop */
215 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
218 /* Get j neighbor index, and coordinate index */
223 j_coord_offsetA = DIM*jnrA;
224 j_coord_offsetB = DIM*jnrB;
225 j_coord_offsetC = DIM*jnrC;
226 j_coord_offsetD = DIM*jnrD;
228 /* load j atom coordinates */
229 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
230 x+j_coord_offsetC,x+j_coord_offsetD,
231 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
233 /* Calculate displacement vector */
234 dx00 = _mm_sub_ps(ix0,jx0);
235 dy00 = _mm_sub_ps(iy0,jy0);
236 dz00 = _mm_sub_ps(iz0,jz0);
237 dx01 = _mm_sub_ps(ix0,jx1);
238 dy01 = _mm_sub_ps(iy0,jy1);
239 dz01 = _mm_sub_ps(iz0,jz1);
240 dx02 = _mm_sub_ps(ix0,jx2);
241 dy02 = _mm_sub_ps(iy0,jy2);
242 dz02 = _mm_sub_ps(iz0,jz2);
243 dx10 = _mm_sub_ps(ix1,jx0);
244 dy10 = _mm_sub_ps(iy1,jy0);
245 dz10 = _mm_sub_ps(iz1,jz0);
246 dx11 = _mm_sub_ps(ix1,jx1);
247 dy11 = _mm_sub_ps(iy1,jy1);
248 dz11 = _mm_sub_ps(iz1,jz1);
249 dx12 = _mm_sub_ps(ix1,jx2);
250 dy12 = _mm_sub_ps(iy1,jy2);
251 dz12 = _mm_sub_ps(iz1,jz2);
252 dx20 = _mm_sub_ps(ix2,jx0);
253 dy20 = _mm_sub_ps(iy2,jy0);
254 dz20 = _mm_sub_ps(iz2,jz0);
255 dx21 = _mm_sub_ps(ix2,jx1);
256 dy21 = _mm_sub_ps(iy2,jy1);
257 dz21 = _mm_sub_ps(iz2,jz1);
258 dx22 = _mm_sub_ps(ix2,jx2);
259 dy22 = _mm_sub_ps(iy2,jy2);
260 dz22 = _mm_sub_ps(iz2,jz2);
262 /* Calculate squared distance and things based on it */
263 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
264 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
265 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
266 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
267 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
268 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
269 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
270 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
271 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
273 rinv00 = gmx_mm_invsqrt_ps(rsq00);
274 rinv01 = gmx_mm_invsqrt_ps(rsq01);
275 rinv02 = gmx_mm_invsqrt_ps(rsq02);
276 rinv10 = gmx_mm_invsqrt_ps(rsq10);
277 rinv11 = gmx_mm_invsqrt_ps(rsq11);
278 rinv12 = gmx_mm_invsqrt_ps(rsq12);
279 rinv20 = gmx_mm_invsqrt_ps(rsq20);
280 rinv21 = gmx_mm_invsqrt_ps(rsq21);
281 rinv22 = gmx_mm_invsqrt_ps(rsq22);
283 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
284 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
285 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
286 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
287 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
288 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
289 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
290 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
291 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
293 fjx0 = _mm_setzero_ps();
294 fjy0 = _mm_setzero_ps();
295 fjz0 = _mm_setzero_ps();
296 fjx1 = _mm_setzero_ps();
297 fjy1 = _mm_setzero_ps();
298 fjz1 = _mm_setzero_ps();
299 fjx2 = _mm_setzero_ps();
300 fjy2 = _mm_setzero_ps();
301 fjz2 = _mm_setzero_ps();
303 /**************************
304 * CALCULATE INTERACTIONS *
305 **************************/
307 if (gmx_mm_any_lt(rsq00,rcutoff2))
310 /* REACTION-FIELD ELECTROSTATICS */
311 velec = _mm_mul_ps(qq00,_mm_sub_ps(_mm_macc_ps(krf,rsq00,rinv00),crf));
312 felec = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
314 /* LENNARD-JONES DISPERSION/REPULSION */
316 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
317 vvdw6 = _mm_mul_ps(c6_00,rinvsix);
318 vvdw12 = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
319 vvdw = _mm_msub_ps(_mm_nmacc_ps(c12_00,_mm_mul_ps(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
320 _mm_mul_ps( _mm_nmacc_ps(c6_00,sh_vdw_invrcut6,vvdw6),one_sixth));
321 fvdw = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
323 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
325 /* Update potential sum for this i atom from the interaction with this j atom. */
326 velec = _mm_and_ps(velec,cutoff_mask);
327 velecsum = _mm_add_ps(velecsum,velec);
328 vvdw = _mm_and_ps(vvdw,cutoff_mask);
329 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
331 fscal = _mm_add_ps(felec,fvdw);
333 fscal = _mm_and_ps(fscal,cutoff_mask);
335 /* Update vectorial force */
336 fix0 = _mm_macc_ps(dx00,fscal,fix0);
337 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
338 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
340 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
341 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
342 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
346 /**************************
347 * CALCULATE INTERACTIONS *
348 **************************/
350 if (gmx_mm_any_lt(rsq01,rcutoff2))
353 /* REACTION-FIELD ELECTROSTATICS */
354 velec = _mm_mul_ps(qq01,_mm_sub_ps(_mm_macc_ps(krf,rsq01,rinv01),crf));
355 felec = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
357 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
359 /* Update potential sum for this i atom from the interaction with this j atom. */
360 velec = _mm_and_ps(velec,cutoff_mask);
361 velecsum = _mm_add_ps(velecsum,velec);
365 fscal = _mm_and_ps(fscal,cutoff_mask);
367 /* Update vectorial force */
368 fix0 = _mm_macc_ps(dx01,fscal,fix0);
369 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
370 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
372 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
373 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
374 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
378 /**************************
379 * CALCULATE INTERACTIONS *
380 **************************/
382 if (gmx_mm_any_lt(rsq02,rcutoff2))
385 /* REACTION-FIELD ELECTROSTATICS */
386 velec = _mm_mul_ps(qq02,_mm_sub_ps(_mm_macc_ps(krf,rsq02,rinv02),crf));
387 felec = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
389 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
391 /* Update potential sum for this i atom from the interaction with this j atom. */
392 velec = _mm_and_ps(velec,cutoff_mask);
393 velecsum = _mm_add_ps(velecsum,velec);
397 fscal = _mm_and_ps(fscal,cutoff_mask);
399 /* Update vectorial force */
400 fix0 = _mm_macc_ps(dx02,fscal,fix0);
401 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
402 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
404 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
405 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
406 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
410 /**************************
411 * CALCULATE INTERACTIONS *
412 **************************/
414 if (gmx_mm_any_lt(rsq10,rcutoff2))
417 /* REACTION-FIELD ELECTROSTATICS */
418 velec = _mm_mul_ps(qq10,_mm_sub_ps(_mm_macc_ps(krf,rsq10,rinv10),crf));
419 felec = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
421 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
423 /* Update potential sum for this i atom from the interaction with this j atom. */
424 velec = _mm_and_ps(velec,cutoff_mask);
425 velecsum = _mm_add_ps(velecsum,velec);
429 fscal = _mm_and_ps(fscal,cutoff_mask);
431 /* Update vectorial force */
432 fix1 = _mm_macc_ps(dx10,fscal,fix1);
433 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
434 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
436 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
437 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
438 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
442 /**************************
443 * CALCULATE INTERACTIONS *
444 **************************/
446 if (gmx_mm_any_lt(rsq11,rcutoff2))
449 /* REACTION-FIELD ELECTROSTATICS */
450 velec = _mm_mul_ps(qq11,_mm_sub_ps(_mm_macc_ps(krf,rsq11,rinv11),crf));
451 felec = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
453 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
455 /* Update potential sum for this i atom from the interaction with this j atom. */
456 velec = _mm_and_ps(velec,cutoff_mask);
457 velecsum = _mm_add_ps(velecsum,velec);
461 fscal = _mm_and_ps(fscal,cutoff_mask);
463 /* Update vectorial force */
464 fix1 = _mm_macc_ps(dx11,fscal,fix1);
465 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
466 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
468 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
469 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
470 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
474 /**************************
475 * CALCULATE INTERACTIONS *
476 **************************/
478 if (gmx_mm_any_lt(rsq12,rcutoff2))
481 /* REACTION-FIELD ELECTROSTATICS */
482 velec = _mm_mul_ps(qq12,_mm_sub_ps(_mm_macc_ps(krf,rsq12,rinv12),crf));
483 felec = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
485 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
487 /* Update potential sum for this i atom from the interaction with this j atom. */
488 velec = _mm_and_ps(velec,cutoff_mask);
489 velecsum = _mm_add_ps(velecsum,velec);
493 fscal = _mm_and_ps(fscal,cutoff_mask);
495 /* Update vectorial force */
496 fix1 = _mm_macc_ps(dx12,fscal,fix1);
497 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
498 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
500 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
501 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
502 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
506 /**************************
507 * CALCULATE INTERACTIONS *
508 **************************/
510 if (gmx_mm_any_lt(rsq20,rcutoff2))
513 /* REACTION-FIELD ELECTROSTATICS */
514 velec = _mm_mul_ps(qq20,_mm_sub_ps(_mm_macc_ps(krf,rsq20,rinv20),crf));
515 felec = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
517 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
519 /* Update potential sum for this i atom from the interaction with this j atom. */
520 velec = _mm_and_ps(velec,cutoff_mask);
521 velecsum = _mm_add_ps(velecsum,velec);
525 fscal = _mm_and_ps(fscal,cutoff_mask);
527 /* Update vectorial force */
528 fix2 = _mm_macc_ps(dx20,fscal,fix2);
529 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
530 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
532 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
533 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
534 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
538 /**************************
539 * CALCULATE INTERACTIONS *
540 **************************/
542 if (gmx_mm_any_lt(rsq21,rcutoff2))
545 /* REACTION-FIELD ELECTROSTATICS */
546 velec = _mm_mul_ps(qq21,_mm_sub_ps(_mm_macc_ps(krf,rsq21,rinv21),crf));
547 felec = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
549 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
551 /* Update potential sum for this i atom from the interaction with this j atom. */
552 velec = _mm_and_ps(velec,cutoff_mask);
553 velecsum = _mm_add_ps(velecsum,velec);
557 fscal = _mm_and_ps(fscal,cutoff_mask);
559 /* Update vectorial force */
560 fix2 = _mm_macc_ps(dx21,fscal,fix2);
561 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
562 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
564 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
565 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
566 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
570 /**************************
571 * CALCULATE INTERACTIONS *
572 **************************/
574 if (gmx_mm_any_lt(rsq22,rcutoff2))
577 /* REACTION-FIELD ELECTROSTATICS */
578 velec = _mm_mul_ps(qq22,_mm_sub_ps(_mm_macc_ps(krf,rsq22,rinv22),crf));
579 felec = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
581 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
583 /* Update potential sum for this i atom from the interaction with this j atom. */
584 velec = _mm_and_ps(velec,cutoff_mask);
585 velecsum = _mm_add_ps(velecsum,velec);
589 fscal = _mm_and_ps(fscal,cutoff_mask);
591 /* Update vectorial force */
592 fix2 = _mm_macc_ps(dx22,fscal,fix2);
593 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
594 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
596 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
597 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
598 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
602 fjptrA = f+j_coord_offsetA;
603 fjptrB = f+j_coord_offsetB;
604 fjptrC = f+j_coord_offsetC;
605 fjptrD = f+j_coord_offsetD;
607 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
608 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
610 /* Inner loop uses 369 flops */
616 /* Get j neighbor index, and coordinate index */
617 jnrlistA = jjnr[jidx];
618 jnrlistB = jjnr[jidx+1];
619 jnrlistC = jjnr[jidx+2];
620 jnrlistD = jjnr[jidx+3];
621 /* Sign of each element will be negative for non-real atoms.
622 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
623 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
625 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
626 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
627 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
628 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
629 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
630 j_coord_offsetA = DIM*jnrA;
631 j_coord_offsetB = DIM*jnrB;
632 j_coord_offsetC = DIM*jnrC;
633 j_coord_offsetD = DIM*jnrD;
635 /* load j atom coordinates */
636 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
637 x+j_coord_offsetC,x+j_coord_offsetD,
638 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
640 /* Calculate displacement vector */
641 dx00 = _mm_sub_ps(ix0,jx0);
642 dy00 = _mm_sub_ps(iy0,jy0);
643 dz00 = _mm_sub_ps(iz0,jz0);
644 dx01 = _mm_sub_ps(ix0,jx1);
645 dy01 = _mm_sub_ps(iy0,jy1);
646 dz01 = _mm_sub_ps(iz0,jz1);
647 dx02 = _mm_sub_ps(ix0,jx2);
648 dy02 = _mm_sub_ps(iy0,jy2);
649 dz02 = _mm_sub_ps(iz0,jz2);
650 dx10 = _mm_sub_ps(ix1,jx0);
651 dy10 = _mm_sub_ps(iy1,jy0);
652 dz10 = _mm_sub_ps(iz1,jz0);
653 dx11 = _mm_sub_ps(ix1,jx1);
654 dy11 = _mm_sub_ps(iy1,jy1);
655 dz11 = _mm_sub_ps(iz1,jz1);
656 dx12 = _mm_sub_ps(ix1,jx2);
657 dy12 = _mm_sub_ps(iy1,jy2);
658 dz12 = _mm_sub_ps(iz1,jz2);
659 dx20 = _mm_sub_ps(ix2,jx0);
660 dy20 = _mm_sub_ps(iy2,jy0);
661 dz20 = _mm_sub_ps(iz2,jz0);
662 dx21 = _mm_sub_ps(ix2,jx1);
663 dy21 = _mm_sub_ps(iy2,jy1);
664 dz21 = _mm_sub_ps(iz2,jz1);
665 dx22 = _mm_sub_ps(ix2,jx2);
666 dy22 = _mm_sub_ps(iy2,jy2);
667 dz22 = _mm_sub_ps(iz2,jz2);
669 /* Calculate squared distance and things based on it */
670 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
671 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
672 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
673 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
674 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
675 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
676 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
677 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
678 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
680 rinv00 = gmx_mm_invsqrt_ps(rsq00);
681 rinv01 = gmx_mm_invsqrt_ps(rsq01);
682 rinv02 = gmx_mm_invsqrt_ps(rsq02);
683 rinv10 = gmx_mm_invsqrt_ps(rsq10);
684 rinv11 = gmx_mm_invsqrt_ps(rsq11);
685 rinv12 = gmx_mm_invsqrt_ps(rsq12);
686 rinv20 = gmx_mm_invsqrt_ps(rsq20);
687 rinv21 = gmx_mm_invsqrt_ps(rsq21);
688 rinv22 = gmx_mm_invsqrt_ps(rsq22);
690 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
691 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
692 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
693 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
694 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
695 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
696 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
697 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
698 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
700 fjx0 = _mm_setzero_ps();
701 fjy0 = _mm_setzero_ps();
702 fjz0 = _mm_setzero_ps();
703 fjx1 = _mm_setzero_ps();
704 fjy1 = _mm_setzero_ps();
705 fjz1 = _mm_setzero_ps();
706 fjx2 = _mm_setzero_ps();
707 fjy2 = _mm_setzero_ps();
708 fjz2 = _mm_setzero_ps();
710 /**************************
711 * CALCULATE INTERACTIONS *
712 **************************/
714 if (gmx_mm_any_lt(rsq00,rcutoff2))
717 /* REACTION-FIELD ELECTROSTATICS */
718 velec = _mm_mul_ps(qq00,_mm_sub_ps(_mm_macc_ps(krf,rsq00,rinv00),crf));
719 felec = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
721 /* LENNARD-JONES DISPERSION/REPULSION */
723 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
724 vvdw6 = _mm_mul_ps(c6_00,rinvsix);
725 vvdw12 = _mm_mul_ps(c12_00,_mm_mul_ps(rinvsix,rinvsix));
726 vvdw = _mm_msub_ps(_mm_nmacc_ps(c12_00,_mm_mul_ps(sh_vdw_invrcut6,sh_vdw_invrcut6),vvdw12),one_twelfth,
727 _mm_mul_ps( _mm_nmacc_ps(c6_00,sh_vdw_invrcut6,vvdw6),one_sixth));
728 fvdw = _mm_mul_ps(_mm_sub_ps(vvdw12,vvdw6),rinvsq00);
730 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
732 /* Update potential sum for this i atom from the interaction with this j atom. */
733 velec = _mm_and_ps(velec,cutoff_mask);
734 velec = _mm_andnot_ps(dummy_mask,velec);
735 velecsum = _mm_add_ps(velecsum,velec);
736 vvdw = _mm_and_ps(vvdw,cutoff_mask);
737 vvdw = _mm_andnot_ps(dummy_mask,vvdw);
738 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
740 fscal = _mm_add_ps(felec,fvdw);
742 fscal = _mm_and_ps(fscal,cutoff_mask);
744 fscal = _mm_andnot_ps(dummy_mask,fscal);
746 /* Update vectorial force */
747 fix0 = _mm_macc_ps(dx00,fscal,fix0);
748 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
749 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
751 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
752 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
753 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
757 /**************************
758 * CALCULATE INTERACTIONS *
759 **************************/
761 if (gmx_mm_any_lt(rsq01,rcutoff2))
764 /* REACTION-FIELD ELECTROSTATICS */
765 velec = _mm_mul_ps(qq01,_mm_sub_ps(_mm_macc_ps(krf,rsq01,rinv01),crf));
766 felec = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
768 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
770 /* Update potential sum for this i atom from the interaction with this j atom. */
771 velec = _mm_and_ps(velec,cutoff_mask);
772 velec = _mm_andnot_ps(dummy_mask,velec);
773 velecsum = _mm_add_ps(velecsum,velec);
777 fscal = _mm_and_ps(fscal,cutoff_mask);
779 fscal = _mm_andnot_ps(dummy_mask,fscal);
781 /* Update vectorial force */
782 fix0 = _mm_macc_ps(dx01,fscal,fix0);
783 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
784 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
786 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
787 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
788 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
792 /**************************
793 * CALCULATE INTERACTIONS *
794 **************************/
796 if (gmx_mm_any_lt(rsq02,rcutoff2))
799 /* REACTION-FIELD ELECTROSTATICS */
800 velec = _mm_mul_ps(qq02,_mm_sub_ps(_mm_macc_ps(krf,rsq02,rinv02),crf));
801 felec = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
803 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
805 /* Update potential sum for this i atom from the interaction with this j atom. */
806 velec = _mm_and_ps(velec,cutoff_mask);
807 velec = _mm_andnot_ps(dummy_mask,velec);
808 velecsum = _mm_add_ps(velecsum,velec);
812 fscal = _mm_and_ps(fscal,cutoff_mask);
814 fscal = _mm_andnot_ps(dummy_mask,fscal);
816 /* Update vectorial force */
817 fix0 = _mm_macc_ps(dx02,fscal,fix0);
818 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
819 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
821 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
822 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
823 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
827 /**************************
828 * CALCULATE INTERACTIONS *
829 **************************/
831 if (gmx_mm_any_lt(rsq10,rcutoff2))
834 /* REACTION-FIELD ELECTROSTATICS */
835 velec = _mm_mul_ps(qq10,_mm_sub_ps(_mm_macc_ps(krf,rsq10,rinv10),crf));
836 felec = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
838 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
840 /* Update potential sum for this i atom from the interaction with this j atom. */
841 velec = _mm_and_ps(velec,cutoff_mask);
842 velec = _mm_andnot_ps(dummy_mask,velec);
843 velecsum = _mm_add_ps(velecsum,velec);
847 fscal = _mm_and_ps(fscal,cutoff_mask);
849 fscal = _mm_andnot_ps(dummy_mask,fscal);
851 /* Update vectorial force */
852 fix1 = _mm_macc_ps(dx10,fscal,fix1);
853 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
854 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
856 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
857 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
858 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
862 /**************************
863 * CALCULATE INTERACTIONS *
864 **************************/
866 if (gmx_mm_any_lt(rsq11,rcutoff2))
869 /* REACTION-FIELD ELECTROSTATICS */
870 velec = _mm_mul_ps(qq11,_mm_sub_ps(_mm_macc_ps(krf,rsq11,rinv11),crf));
871 felec = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
873 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
875 /* Update potential sum for this i atom from the interaction with this j atom. */
876 velec = _mm_and_ps(velec,cutoff_mask);
877 velec = _mm_andnot_ps(dummy_mask,velec);
878 velecsum = _mm_add_ps(velecsum,velec);
882 fscal = _mm_and_ps(fscal,cutoff_mask);
884 fscal = _mm_andnot_ps(dummy_mask,fscal);
886 /* Update vectorial force */
887 fix1 = _mm_macc_ps(dx11,fscal,fix1);
888 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
889 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
891 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
892 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
893 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
897 /**************************
898 * CALCULATE INTERACTIONS *
899 **************************/
901 if (gmx_mm_any_lt(rsq12,rcutoff2))
904 /* REACTION-FIELD ELECTROSTATICS */
905 velec = _mm_mul_ps(qq12,_mm_sub_ps(_mm_macc_ps(krf,rsq12,rinv12),crf));
906 felec = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
908 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
910 /* Update potential sum for this i atom from the interaction with this j atom. */
911 velec = _mm_and_ps(velec,cutoff_mask);
912 velec = _mm_andnot_ps(dummy_mask,velec);
913 velecsum = _mm_add_ps(velecsum,velec);
917 fscal = _mm_and_ps(fscal,cutoff_mask);
919 fscal = _mm_andnot_ps(dummy_mask,fscal);
921 /* Update vectorial force */
922 fix1 = _mm_macc_ps(dx12,fscal,fix1);
923 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
924 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
926 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
927 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
928 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
932 /**************************
933 * CALCULATE INTERACTIONS *
934 **************************/
936 if (gmx_mm_any_lt(rsq20,rcutoff2))
939 /* REACTION-FIELD ELECTROSTATICS */
940 velec = _mm_mul_ps(qq20,_mm_sub_ps(_mm_macc_ps(krf,rsq20,rinv20),crf));
941 felec = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
943 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
945 /* Update potential sum for this i atom from the interaction with this j atom. */
946 velec = _mm_and_ps(velec,cutoff_mask);
947 velec = _mm_andnot_ps(dummy_mask,velec);
948 velecsum = _mm_add_ps(velecsum,velec);
952 fscal = _mm_and_ps(fscal,cutoff_mask);
954 fscal = _mm_andnot_ps(dummy_mask,fscal);
956 /* Update vectorial force */
957 fix2 = _mm_macc_ps(dx20,fscal,fix2);
958 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
959 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
961 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
962 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
963 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
967 /**************************
968 * CALCULATE INTERACTIONS *
969 **************************/
971 if (gmx_mm_any_lt(rsq21,rcutoff2))
974 /* REACTION-FIELD ELECTROSTATICS */
975 velec = _mm_mul_ps(qq21,_mm_sub_ps(_mm_macc_ps(krf,rsq21,rinv21),crf));
976 felec = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
978 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
980 /* Update potential sum for this i atom from the interaction with this j atom. */
981 velec = _mm_and_ps(velec,cutoff_mask);
982 velec = _mm_andnot_ps(dummy_mask,velec);
983 velecsum = _mm_add_ps(velecsum,velec);
987 fscal = _mm_and_ps(fscal,cutoff_mask);
989 fscal = _mm_andnot_ps(dummy_mask,fscal);
991 /* Update vectorial force */
992 fix2 = _mm_macc_ps(dx21,fscal,fix2);
993 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
994 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
996 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
997 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
998 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1002 /**************************
1003 * CALCULATE INTERACTIONS *
1004 **************************/
1006 if (gmx_mm_any_lt(rsq22,rcutoff2))
1009 /* REACTION-FIELD ELECTROSTATICS */
1010 velec = _mm_mul_ps(qq22,_mm_sub_ps(_mm_macc_ps(krf,rsq22,rinv22),crf));
1011 felec = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
1013 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
1015 /* Update potential sum for this i atom from the interaction with this j atom. */
1016 velec = _mm_and_ps(velec,cutoff_mask);
1017 velec = _mm_andnot_ps(dummy_mask,velec);
1018 velecsum = _mm_add_ps(velecsum,velec);
1022 fscal = _mm_and_ps(fscal,cutoff_mask);
1024 fscal = _mm_andnot_ps(dummy_mask,fscal);
1026 /* Update vectorial force */
1027 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1028 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1029 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1031 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1032 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1033 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1037 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1038 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1039 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1040 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1042 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1043 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1045 /* Inner loop uses 369 flops */
1048 /* End of innermost loop */
1050 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1051 f+i_coord_offset,fshift+i_shift_offset);
1054 /* Update potential energies */
1055 gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1056 gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1058 /* Increment number of inner iterations */
1059 inneriter += j_index_end - j_index_start;
1061 /* Outer loop uses 20 flops */
1064 /* Increment number of outer iterations */
1067 /* Update outer/inner flops */
1069 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*369);
1072 * Gromacs nonbonded kernel: nb_kernel_ElecRFCut_VdwLJSh_GeomW3W3_F_avx_128_fma_single
1073 * Electrostatics interaction: ReactionField
1074 * VdW interaction: LennardJones
1075 * Geometry: Water3-Water3
1076 * Calculate force/pot: Force
1079 nb_kernel_ElecRFCut_VdwLJSh_GeomW3W3_F_avx_128_fma_single
1080 (t_nblist * gmx_restrict nlist,
1081 rvec * gmx_restrict xx,
1082 rvec * gmx_restrict ff,
1083 t_forcerec * gmx_restrict fr,
1084 t_mdatoms * gmx_restrict mdatoms,
1085 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1086 t_nrnb * gmx_restrict nrnb)
1088 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1089 * just 0 for non-waters.
1090 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1091 * jnr indices corresponding to data put in the four positions in the SIMD register.
1093 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1094 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1095 int jnrA,jnrB,jnrC,jnrD;
1096 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1097 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1098 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1099 real rcutoff_scalar;
1100 real *shiftvec,*fshift,*x,*f;
1101 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1102 real scratch[4*DIM];
1103 __m128 fscal,rcutoff,rcutoff2,jidxall;
1105 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1107 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1109 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1110 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1111 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1112 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1113 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1114 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1115 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1116 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1117 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1118 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1119 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1120 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1121 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1122 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1123 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1124 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1125 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
1128 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1131 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
1132 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
1133 __m128 dummy_mask,cutoff_mask;
1134 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1135 __m128 one = _mm_set1_ps(1.0);
1136 __m128 two = _mm_set1_ps(2.0);
1142 jindex = nlist->jindex;
1144 shiftidx = nlist->shift;
1146 shiftvec = fr->shift_vec[0];
1147 fshift = fr->fshift[0];
1148 facel = _mm_set1_ps(fr->epsfac);
1149 charge = mdatoms->chargeA;
1150 krf = _mm_set1_ps(fr->ic->k_rf);
1151 krf2 = _mm_set1_ps(fr->ic->k_rf*2.0);
1152 crf = _mm_set1_ps(fr->ic->c_rf);
1153 nvdwtype = fr->ntype;
1154 vdwparam = fr->nbfp;
1155 vdwtype = mdatoms->typeA;
1157 /* Setup water-specific parameters */
1158 inr = nlist->iinr[0];
1159 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1160 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1161 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1162 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
1164 jq0 = _mm_set1_ps(charge[inr+0]);
1165 jq1 = _mm_set1_ps(charge[inr+1]);
1166 jq2 = _mm_set1_ps(charge[inr+2]);
1167 vdwjidx0A = 2*vdwtype[inr+0];
1168 qq00 = _mm_mul_ps(iq0,jq0);
1169 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1170 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1171 qq01 = _mm_mul_ps(iq0,jq1);
1172 qq02 = _mm_mul_ps(iq0,jq2);
1173 qq10 = _mm_mul_ps(iq1,jq0);
1174 qq11 = _mm_mul_ps(iq1,jq1);
1175 qq12 = _mm_mul_ps(iq1,jq2);
1176 qq20 = _mm_mul_ps(iq2,jq0);
1177 qq21 = _mm_mul_ps(iq2,jq1);
1178 qq22 = _mm_mul_ps(iq2,jq2);
1180 /* When we use explicit cutoffs the value must be identical for elec and VdW, so use elec as an arbitrary choice */
1181 rcutoff_scalar = fr->rcoulomb;
1182 rcutoff = _mm_set1_ps(rcutoff_scalar);
1183 rcutoff2 = _mm_mul_ps(rcutoff,rcutoff);
1185 sh_vdw_invrcut6 = _mm_set1_ps(fr->ic->sh_invrc6);
1186 rvdw = _mm_set1_ps(fr->rvdw);
1188 /* Avoid stupid compiler warnings */
1189 jnrA = jnrB = jnrC = jnrD = 0;
1190 j_coord_offsetA = 0;
1191 j_coord_offsetB = 0;
1192 j_coord_offsetC = 0;
1193 j_coord_offsetD = 0;
1198 for(iidx=0;iidx<4*DIM;iidx++)
1200 scratch[iidx] = 0.0;
1203 /* Start outer loop over neighborlists */
1204 for(iidx=0; iidx<nri; iidx++)
1206 /* Load shift vector for this list */
1207 i_shift_offset = DIM*shiftidx[iidx];
1209 /* Load limits for loop over neighbors */
1210 j_index_start = jindex[iidx];
1211 j_index_end = jindex[iidx+1];
1213 /* Get outer coordinate index */
1215 i_coord_offset = DIM*inr;
1217 /* Load i particle coords and add shift vector */
1218 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1219 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1221 fix0 = _mm_setzero_ps();
1222 fiy0 = _mm_setzero_ps();
1223 fiz0 = _mm_setzero_ps();
1224 fix1 = _mm_setzero_ps();
1225 fiy1 = _mm_setzero_ps();
1226 fiz1 = _mm_setzero_ps();
1227 fix2 = _mm_setzero_ps();
1228 fiy2 = _mm_setzero_ps();
1229 fiz2 = _mm_setzero_ps();
1231 /* Start inner kernel loop */
1232 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1235 /* Get j neighbor index, and coordinate index */
1237 jnrB = jjnr[jidx+1];
1238 jnrC = jjnr[jidx+2];
1239 jnrD = jjnr[jidx+3];
1240 j_coord_offsetA = DIM*jnrA;
1241 j_coord_offsetB = DIM*jnrB;
1242 j_coord_offsetC = DIM*jnrC;
1243 j_coord_offsetD = DIM*jnrD;
1245 /* load j atom coordinates */
1246 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1247 x+j_coord_offsetC,x+j_coord_offsetD,
1248 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1250 /* Calculate displacement vector */
1251 dx00 = _mm_sub_ps(ix0,jx0);
1252 dy00 = _mm_sub_ps(iy0,jy0);
1253 dz00 = _mm_sub_ps(iz0,jz0);
1254 dx01 = _mm_sub_ps(ix0,jx1);
1255 dy01 = _mm_sub_ps(iy0,jy1);
1256 dz01 = _mm_sub_ps(iz0,jz1);
1257 dx02 = _mm_sub_ps(ix0,jx2);
1258 dy02 = _mm_sub_ps(iy0,jy2);
1259 dz02 = _mm_sub_ps(iz0,jz2);
1260 dx10 = _mm_sub_ps(ix1,jx0);
1261 dy10 = _mm_sub_ps(iy1,jy0);
1262 dz10 = _mm_sub_ps(iz1,jz0);
1263 dx11 = _mm_sub_ps(ix1,jx1);
1264 dy11 = _mm_sub_ps(iy1,jy1);
1265 dz11 = _mm_sub_ps(iz1,jz1);
1266 dx12 = _mm_sub_ps(ix1,jx2);
1267 dy12 = _mm_sub_ps(iy1,jy2);
1268 dz12 = _mm_sub_ps(iz1,jz2);
1269 dx20 = _mm_sub_ps(ix2,jx0);
1270 dy20 = _mm_sub_ps(iy2,jy0);
1271 dz20 = _mm_sub_ps(iz2,jz0);
1272 dx21 = _mm_sub_ps(ix2,jx1);
1273 dy21 = _mm_sub_ps(iy2,jy1);
1274 dz21 = _mm_sub_ps(iz2,jz1);
1275 dx22 = _mm_sub_ps(ix2,jx2);
1276 dy22 = _mm_sub_ps(iy2,jy2);
1277 dz22 = _mm_sub_ps(iz2,jz2);
1279 /* Calculate squared distance and things based on it */
1280 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1281 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1282 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1283 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1284 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1285 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1286 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1287 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1288 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1290 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1291 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1292 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1293 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1294 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1295 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1296 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1297 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1298 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1300 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1301 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1302 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1303 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1304 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1305 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1306 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1307 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1308 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1310 fjx0 = _mm_setzero_ps();
1311 fjy0 = _mm_setzero_ps();
1312 fjz0 = _mm_setzero_ps();
1313 fjx1 = _mm_setzero_ps();
1314 fjy1 = _mm_setzero_ps();
1315 fjz1 = _mm_setzero_ps();
1316 fjx2 = _mm_setzero_ps();
1317 fjy2 = _mm_setzero_ps();
1318 fjz2 = _mm_setzero_ps();
1320 /**************************
1321 * CALCULATE INTERACTIONS *
1322 **************************/
1324 if (gmx_mm_any_lt(rsq00,rcutoff2))
1327 /* REACTION-FIELD ELECTROSTATICS */
1328 felec = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
1330 /* LENNARD-JONES DISPERSION/REPULSION */
1332 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1333 fvdw = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1335 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
1337 fscal = _mm_add_ps(felec,fvdw);
1339 fscal = _mm_and_ps(fscal,cutoff_mask);
1341 /* Update vectorial force */
1342 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1343 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1344 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1346 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1347 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1348 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1352 /**************************
1353 * CALCULATE INTERACTIONS *
1354 **************************/
1356 if (gmx_mm_any_lt(rsq01,rcutoff2))
1359 /* REACTION-FIELD ELECTROSTATICS */
1360 felec = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
1362 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
1366 fscal = _mm_and_ps(fscal,cutoff_mask);
1368 /* Update vectorial force */
1369 fix0 = _mm_macc_ps(dx01,fscal,fix0);
1370 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
1371 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
1373 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
1374 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
1375 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
1379 /**************************
1380 * CALCULATE INTERACTIONS *
1381 **************************/
1383 if (gmx_mm_any_lt(rsq02,rcutoff2))
1386 /* REACTION-FIELD ELECTROSTATICS */
1387 felec = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
1389 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
1393 fscal = _mm_and_ps(fscal,cutoff_mask);
1395 /* Update vectorial force */
1396 fix0 = _mm_macc_ps(dx02,fscal,fix0);
1397 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
1398 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
1400 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
1401 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
1402 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
1406 /**************************
1407 * CALCULATE INTERACTIONS *
1408 **************************/
1410 if (gmx_mm_any_lt(rsq10,rcutoff2))
1413 /* REACTION-FIELD ELECTROSTATICS */
1414 felec = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
1416 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
1420 fscal = _mm_and_ps(fscal,cutoff_mask);
1422 /* Update vectorial force */
1423 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1424 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1425 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1427 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1428 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1429 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1433 /**************************
1434 * CALCULATE INTERACTIONS *
1435 **************************/
1437 if (gmx_mm_any_lt(rsq11,rcutoff2))
1440 /* REACTION-FIELD ELECTROSTATICS */
1441 felec = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
1443 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
1447 fscal = _mm_and_ps(fscal,cutoff_mask);
1449 /* Update vectorial force */
1450 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1451 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1452 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1454 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1455 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1456 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1460 /**************************
1461 * CALCULATE INTERACTIONS *
1462 **************************/
1464 if (gmx_mm_any_lt(rsq12,rcutoff2))
1467 /* REACTION-FIELD ELECTROSTATICS */
1468 felec = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
1470 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
1474 fscal = _mm_and_ps(fscal,cutoff_mask);
1476 /* Update vectorial force */
1477 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1478 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1479 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1481 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1482 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1483 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1487 /**************************
1488 * CALCULATE INTERACTIONS *
1489 **************************/
1491 if (gmx_mm_any_lt(rsq20,rcutoff2))
1494 /* REACTION-FIELD ELECTROSTATICS */
1495 felec = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
1497 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
1501 fscal = _mm_and_ps(fscal,cutoff_mask);
1503 /* Update vectorial force */
1504 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1505 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1506 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1508 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1509 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1510 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1514 /**************************
1515 * CALCULATE INTERACTIONS *
1516 **************************/
1518 if (gmx_mm_any_lt(rsq21,rcutoff2))
1521 /* REACTION-FIELD ELECTROSTATICS */
1522 felec = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
1524 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
1528 fscal = _mm_and_ps(fscal,cutoff_mask);
1530 /* Update vectorial force */
1531 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1532 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1533 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1535 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1536 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1537 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1541 /**************************
1542 * CALCULATE INTERACTIONS *
1543 **************************/
1545 if (gmx_mm_any_lt(rsq22,rcutoff2))
1548 /* REACTION-FIELD ELECTROSTATICS */
1549 felec = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
1551 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
1555 fscal = _mm_and_ps(fscal,cutoff_mask);
1557 /* Update vectorial force */
1558 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1559 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1560 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1562 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1563 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1564 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1568 fjptrA = f+j_coord_offsetA;
1569 fjptrB = f+j_coord_offsetB;
1570 fjptrC = f+j_coord_offsetC;
1571 fjptrD = f+j_coord_offsetD;
1573 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1574 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1576 /* Inner loop uses 304 flops */
1579 if(jidx<j_index_end)
1582 /* Get j neighbor index, and coordinate index */
1583 jnrlistA = jjnr[jidx];
1584 jnrlistB = jjnr[jidx+1];
1585 jnrlistC = jjnr[jidx+2];
1586 jnrlistD = jjnr[jidx+3];
1587 /* Sign of each element will be negative for non-real atoms.
1588 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1589 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1591 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1592 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1593 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1594 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1595 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1596 j_coord_offsetA = DIM*jnrA;
1597 j_coord_offsetB = DIM*jnrB;
1598 j_coord_offsetC = DIM*jnrC;
1599 j_coord_offsetD = DIM*jnrD;
1601 /* load j atom coordinates */
1602 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1603 x+j_coord_offsetC,x+j_coord_offsetD,
1604 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1606 /* Calculate displacement vector */
1607 dx00 = _mm_sub_ps(ix0,jx0);
1608 dy00 = _mm_sub_ps(iy0,jy0);
1609 dz00 = _mm_sub_ps(iz0,jz0);
1610 dx01 = _mm_sub_ps(ix0,jx1);
1611 dy01 = _mm_sub_ps(iy0,jy1);
1612 dz01 = _mm_sub_ps(iz0,jz1);
1613 dx02 = _mm_sub_ps(ix0,jx2);
1614 dy02 = _mm_sub_ps(iy0,jy2);
1615 dz02 = _mm_sub_ps(iz0,jz2);
1616 dx10 = _mm_sub_ps(ix1,jx0);
1617 dy10 = _mm_sub_ps(iy1,jy0);
1618 dz10 = _mm_sub_ps(iz1,jz0);
1619 dx11 = _mm_sub_ps(ix1,jx1);
1620 dy11 = _mm_sub_ps(iy1,jy1);
1621 dz11 = _mm_sub_ps(iz1,jz1);
1622 dx12 = _mm_sub_ps(ix1,jx2);
1623 dy12 = _mm_sub_ps(iy1,jy2);
1624 dz12 = _mm_sub_ps(iz1,jz2);
1625 dx20 = _mm_sub_ps(ix2,jx0);
1626 dy20 = _mm_sub_ps(iy2,jy0);
1627 dz20 = _mm_sub_ps(iz2,jz0);
1628 dx21 = _mm_sub_ps(ix2,jx1);
1629 dy21 = _mm_sub_ps(iy2,jy1);
1630 dz21 = _mm_sub_ps(iz2,jz1);
1631 dx22 = _mm_sub_ps(ix2,jx2);
1632 dy22 = _mm_sub_ps(iy2,jy2);
1633 dz22 = _mm_sub_ps(iz2,jz2);
1635 /* Calculate squared distance and things based on it */
1636 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1637 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1638 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1639 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1640 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1641 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1642 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1643 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1644 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1646 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1647 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1648 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1649 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1650 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1651 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1652 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1653 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1654 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1656 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1657 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1658 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1659 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1660 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1661 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1662 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1663 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1664 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1666 fjx0 = _mm_setzero_ps();
1667 fjy0 = _mm_setzero_ps();
1668 fjz0 = _mm_setzero_ps();
1669 fjx1 = _mm_setzero_ps();
1670 fjy1 = _mm_setzero_ps();
1671 fjz1 = _mm_setzero_ps();
1672 fjx2 = _mm_setzero_ps();
1673 fjy2 = _mm_setzero_ps();
1674 fjz2 = _mm_setzero_ps();
1676 /**************************
1677 * CALCULATE INTERACTIONS *
1678 **************************/
1680 if (gmx_mm_any_lt(rsq00,rcutoff2))
1683 /* REACTION-FIELD ELECTROSTATICS */
1684 felec = _mm_mul_ps(qq00,_mm_msub_ps(rinv00,rinvsq00,krf2));
1686 /* LENNARD-JONES DISPERSION/REPULSION */
1688 rinvsix = _mm_mul_ps(_mm_mul_ps(rinvsq00,rinvsq00),rinvsq00);
1689 fvdw = _mm_mul_ps(_mm_msub_ps(c12_00,rinvsix,c6_00),_mm_mul_ps(rinvsix,rinvsq00));
1691 cutoff_mask = _mm_cmplt_ps(rsq00,rcutoff2);
1693 fscal = _mm_add_ps(felec,fvdw);
1695 fscal = _mm_and_ps(fscal,cutoff_mask);
1697 fscal = _mm_andnot_ps(dummy_mask,fscal);
1699 /* Update vectorial force */
1700 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1701 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1702 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1704 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1705 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1706 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1710 /**************************
1711 * CALCULATE INTERACTIONS *
1712 **************************/
1714 if (gmx_mm_any_lt(rsq01,rcutoff2))
1717 /* REACTION-FIELD ELECTROSTATICS */
1718 felec = _mm_mul_ps(qq01,_mm_msub_ps(rinv01,rinvsq01,krf2));
1720 cutoff_mask = _mm_cmplt_ps(rsq01,rcutoff2);
1724 fscal = _mm_and_ps(fscal,cutoff_mask);
1726 fscal = _mm_andnot_ps(dummy_mask,fscal);
1728 /* Update vectorial force */
1729 fix0 = _mm_macc_ps(dx01,fscal,fix0);
1730 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
1731 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
1733 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
1734 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
1735 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
1739 /**************************
1740 * CALCULATE INTERACTIONS *
1741 **************************/
1743 if (gmx_mm_any_lt(rsq02,rcutoff2))
1746 /* REACTION-FIELD ELECTROSTATICS */
1747 felec = _mm_mul_ps(qq02,_mm_msub_ps(rinv02,rinvsq02,krf2));
1749 cutoff_mask = _mm_cmplt_ps(rsq02,rcutoff2);
1753 fscal = _mm_and_ps(fscal,cutoff_mask);
1755 fscal = _mm_andnot_ps(dummy_mask,fscal);
1757 /* Update vectorial force */
1758 fix0 = _mm_macc_ps(dx02,fscal,fix0);
1759 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
1760 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
1762 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
1763 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
1764 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
1768 /**************************
1769 * CALCULATE INTERACTIONS *
1770 **************************/
1772 if (gmx_mm_any_lt(rsq10,rcutoff2))
1775 /* REACTION-FIELD ELECTROSTATICS */
1776 felec = _mm_mul_ps(qq10,_mm_msub_ps(rinv10,rinvsq10,krf2));
1778 cutoff_mask = _mm_cmplt_ps(rsq10,rcutoff2);
1782 fscal = _mm_and_ps(fscal,cutoff_mask);
1784 fscal = _mm_andnot_ps(dummy_mask,fscal);
1786 /* Update vectorial force */
1787 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1788 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1789 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1791 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1792 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1793 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1797 /**************************
1798 * CALCULATE INTERACTIONS *
1799 **************************/
1801 if (gmx_mm_any_lt(rsq11,rcutoff2))
1804 /* REACTION-FIELD ELECTROSTATICS */
1805 felec = _mm_mul_ps(qq11,_mm_msub_ps(rinv11,rinvsq11,krf2));
1807 cutoff_mask = _mm_cmplt_ps(rsq11,rcutoff2);
1811 fscal = _mm_and_ps(fscal,cutoff_mask);
1813 fscal = _mm_andnot_ps(dummy_mask,fscal);
1815 /* Update vectorial force */
1816 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1817 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1818 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1820 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1821 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1822 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1826 /**************************
1827 * CALCULATE INTERACTIONS *
1828 **************************/
1830 if (gmx_mm_any_lt(rsq12,rcutoff2))
1833 /* REACTION-FIELD ELECTROSTATICS */
1834 felec = _mm_mul_ps(qq12,_mm_msub_ps(rinv12,rinvsq12,krf2));
1836 cutoff_mask = _mm_cmplt_ps(rsq12,rcutoff2);
1840 fscal = _mm_and_ps(fscal,cutoff_mask);
1842 fscal = _mm_andnot_ps(dummy_mask,fscal);
1844 /* Update vectorial force */
1845 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1846 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1847 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1849 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1850 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1851 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1855 /**************************
1856 * CALCULATE INTERACTIONS *
1857 **************************/
1859 if (gmx_mm_any_lt(rsq20,rcutoff2))
1862 /* REACTION-FIELD ELECTROSTATICS */
1863 felec = _mm_mul_ps(qq20,_mm_msub_ps(rinv20,rinvsq20,krf2));
1865 cutoff_mask = _mm_cmplt_ps(rsq20,rcutoff2);
1869 fscal = _mm_and_ps(fscal,cutoff_mask);
1871 fscal = _mm_andnot_ps(dummy_mask,fscal);
1873 /* Update vectorial force */
1874 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1875 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1876 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1878 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1879 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1880 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1884 /**************************
1885 * CALCULATE INTERACTIONS *
1886 **************************/
1888 if (gmx_mm_any_lt(rsq21,rcutoff2))
1891 /* REACTION-FIELD ELECTROSTATICS */
1892 felec = _mm_mul_ps(qq21,_mm_msub_ps(rinv21,rinvsq21,krf2));
1894 cutoff_mask = _mm_cmplt_ps(rsq21,rcutoff2);
1898 fscal = _mm_and_ps(fscal,cutoff_mask);
1900 fscal = _mm_andnot_ps(dummy_mask,fscal);
1902 /* Update vectorial force */
1903 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1904 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1905 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1907 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1908 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1909 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1913 /**************************
1914 * CALCULATE INTERACTIONS *
1915 **************************/
1917 if (gmx_mm_any_lt(rsq22,rcutoff2))
1920 /* REACTION-FIELD ELECTROSTATICS */
1921 felec = _mm_mul_ps(qq22,_mm_msub_ps(rinv22,rinvsq22,krf2));
1923 cutoff_mask = _mm_cmplt_ps(rsq22,rcutoff2);
1927 fscal = _mm_and_ps(fscal,cutoff_mask);
1929 fscal = _mm_andnot_ps(dummy_mask,fscal);
1931 /* Update vectorial force */
1932 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1933 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1934 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1936 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1937 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1938 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1942 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1943 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1944 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1945 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1947 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1948 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1950 /* Inner loop uses 304 flops */
1953 /* End of innermost loop */
1955 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1956 f+i_coord_offset,fshift+i_shift_offset);
1958 /* Increment number of inner iterations */
1959 inneriter += j_index_end - j_index_start;
1961 /* Outer loop uses 18 flops */
1964 /* Increment number of outer iterations */
1967 /* Update outer/inner flops */
1969 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*304);