2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_128_fma_single kernel generator.
42 #include "../nb_kernel.h"
43 #include "gromacs/legacyheaders/types/simple.h"
44 #include "gromacs/math/vec.h"
45 #include "gromacs/legacyheaders/nrnb.h"
47 #include "gromacs/simd/math_x86_avx_128_fma_single.h"
48 #include "kernelutil_x86_avx_128_fma_single.h"
51 * Gromacs nonbonded kernel: nb_kernel_ElecEw_VdwNone_GeomW3W3_VF_avx_128_fma_single
52 * Electrostatics interaction: Ewald
53 * VdW interaction: None
54 * Geometry: Water3-Water3
55 * Calculate force/pot: PotentialAndForce
58 nb_kernel_ElecEw_VdwNone_GeomW3W3_VF_avx_128_fma_single
59 (t_nblist * gmx_restrict nlist,
60 rvec * gmx_restrict xx,
61 rvec * gmx_restrict ff,
62 t_forcerec * gmx_restrict fr,
63 t_mdatoms * gmx_restrict mdatoms,
64 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
65 t_nrnb * gmx_restrict nrnb)
67 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
68 * just 0 for non-waters.
69 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
70 * jnr indices corresponding to data put in the four positions in the SIMD register.
72 int i_shift_offset,i_coord_offset,outeriter,inneriter;
73 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
74 int jnrA,jnrB,jnrC,jnrD;
75 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
76 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
77 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
79 real *shiftvec,*fshift,*x,*f;
80 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
82 __m128 fscal,rcutoff,rcutoff2,jidxall;
84 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
86 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
88 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
89 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
90 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
91 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
92 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
93 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
94 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
95 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
96 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
97 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
98 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
99 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
100 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
101 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
102 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
103 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
104 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
107 __m128 ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
108 __m128 beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
110 __m128 dummy_mask,cutoff_mask;
111 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
112 __m128 one = _mm_set1_ps(1.0);
113 __m128 two = _mm_set1_ps(2.0);
119 jindex = nlist->jindex;
121 shiftidx = nlist->shift;
123 shiftvec = fr->shift_vec[0];
124 fshift = fr->fshift[0];
125 facel = _mm_set1_ps(fr->epsfac);
126 charge = mdatoms->chargeA;
128 sh_ewald = _mm_set1_ps(fr->ic->sh_ewald);
129 beta = _mm_set1_ps(fr->ic->ewaldcoeff_q);
130 beta2 = _mm_mul_ps(beta,beta);
131 beta3 = _mm_mul_ps(beta,beta2);
132 ewtab = fr->ic->tabq_coul_FDV0;
133 ewtabscale = _mm_set1_ps(fr->ic->tabq_scale);
134 ewtabhalfspace = _mm_set1_ps(0.5/fr->ic->tabq_scale);
136 /* Setup water-specific parameters */
137 inr = nlist->iinr[0];
138 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
139 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
140 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
142 jq0 = _mm_set1_ps(charge[inr+0]);
143 jq1 = _mm_set1_ps(charge[inr+1]);
144 jq2 = _mm_set1_ps(charge[inr+2]);
145 qq00 = _mm_mul_ps(iq0,jq0);
146 qq01 = _mm_mul_ps(iq0,jq1);
147 qq02 = _mm_mul_ps(iq0,jq2);
148 qq10 = _mm_mul_ps(iq1,jq0);
149 qq11 = _mm_mul_ps(iq1,jq1);
150 qq12 = _mm_mul_ps(iq1,jq2);
151 qq20 = _mm_mul_ps(iq2,jq0);
152 qq21 = _mm_mul_ps(iq2,jq1);
153 qq22 = _mm_mul_ps(iq2,jq2);
155 /* Avoid stupid compiler warnings */
156 jnrA = jnrB = jnrC = jnrD = 0;
165 for(iidx=0;iidx<4*DIM;iidx++)
170 /* Start outer loop over neighborlists */
171 for(iidx=0; iidx<nri; iidx++)
173 /* Load shift vector for this list */
174 i_shift_offset = DIM*shiftidx[iidx];
176 /* Load limits for loop over neighbors */
177 j_index_start = jindex[iidx];
178 j_index_end = jindex[iidx+1];
180 /* Get outer coordinate index */
182 i_coord_offset = DIM*inr;
184 /* Load i particle coords and add shift vector */
185 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
186 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
188 fix0 = _mm_setzero_ps();
189 fiy0 = _mm_setzero_ps();
190 fiz0 = _mm_setzero_ps();
191 fix1 = _mm_setzero_ps();
192 fiy1 = _mm_setzero_ps();
193 fiz1 = _mm_setzero_ps();
194 fix2 = _mm_setzero_ps();
195 fiy2 = _mm_setzero_ps();
196 fiz2 = _mm_setzero_ps();
198 /* Reset potential sums */
199 velecsum = _mm_setzero_ps();
201 /* Start inner kernel loop */
202 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
205 /* Get j neighbor index, and coordinate index */
210 j_coord_offsetA = DIM*jnrA;
211 j_coord_offsetB = DIM*jnrB;
212 j_coord_offsetC = DIM*jnrC;
213 j_coord_offsetD = DIM*jnrD;
215 /* load j atom coordinates */
216 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
217 x+j_coord_offsetC,x+j_coord_offsetD,
218 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
220 /* Calculate displacement vector */
221 dx00 = _mm_sub_ps(ix0,jx0);
222 dy00 = _mm_sub_ps(iy0,jy0);
223 dz00 = _mm_sub_ps(iz0,jz0);
224 dx01 = _mm_sub_ps(ix0,jx1);
225 dy01 = _mm_sub_ps(iy0,jy1);
226 dz01 = _mm_sub_ps(iz0,jz1);
227 dx02 = _mm_sub_ps(ix0,jx2);
228 dy02 = _mm_sub_ps(iy0,jy2);
229 dz02 = _mm_sub_ps(iz0,jz2);
230 dx10 = _mm_sub_ps(ix1,jx0);
231 dy10 = _mm_sub_ps(iy1,jy0);
232 dz10 = _mm_sub_ps(iz1,jz0);
233 dx11 = _mm_sub_ps(ix1,jx1);
234 dy11 = _mm_sub_ps(iy1,jy1);
235 dz11 = _mm_sub_ps(iz1,jz1);
236 dx12 = _mm_sub_ps(ix1,jx2);
237 dy12 = _mm_sub_ps(iy1,jy2);
238 dz12 = _mm_sub_ps(iz1,jz2);
239 dx20 = _mm_sub_ps(ix2,jx0);
240 dy20 = _mm_sub_ps(iy2,jy0);
241 dz20 = _mm_sub_ps(iz2,jz0);
242 dx21 = _mm_sub_ps(ix2,jx1);
243 dy21 = _mm_sub_ps(iy2,jy1);
244 dz21 = _mm_sub_ps(iz2,jz1);
245 dx22 = _mm_sub_ps(ix2,jx2);
246 dy22 = _mm_sub_ps(iy2,jy2);
247 dz22 = _mm_sub_ps(iz2,jz2);
249 /* Calculate squared distance and things based on it */
250 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
251 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
252 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
253 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
254 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
255 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
256 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
257 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
258 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
260 rinv00 = gmx_mm_invsqrt_ps(rsq00);
261 rinv01 = gmx_mm_invsqrt_ps(rsq01);
262 rinv02 = gmx_mm_invsqrt_ps(rsq02);
263 rinv10 = gmx_mm_invsqrt_ps(rsq10);
264 rinv11 = gmx_mm_invsqrt_ps(rsq11);
265 rinv12 = gmx_mm_invsqrt_ps(rsq12);
266 rinv20 = gmx_mm_invsqrt_ps(rsq20);
267 rinv21 = gmx_mm_invsqrt_ps(rsq21);
268 rinv22 = gmx_mm_invsqrt_ps(rsq22);
270 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
271 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
272 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
273 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
274 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
275 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
276 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
277 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
278 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
280 fjx0 = _mm_setzero_ps();
281 fjy0 = _mm_setzero_ps();
282 fjz0 = _mm_setzero_ps();
283 fjx1 = _mm_setzero_ps();
284 fjy1 = _mm_setzero_ps();
285 fjz1 = _mm_setzero_ps();
286 fjx2 = _mm_setzero_ps();
287 fjy2 = _mm_setzero_ps();
288 fjz2 = _mm_setzero_ps();
290 /**************************
291 * CALCULATE INTERACTIONS *
292 **************************/
294 r00 = _mm_mul_ps(rsq00,rinv00);
296 /* EWALD ELECTROSTATICS */
298 /* Analytical PME correction */
299 zeta2 = _mm_mul_ps(beta2,rsq00);
300 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
301 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
302 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
303 felec = _mm_mul_ps(qq00,felec);
304 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
305 velec = _mm_nmacc_ps(pmecorrV,beta,rinv00);
306 velec = _mm_mul_ps(qq00,velec);
308 /* Update potential sum for this i atom from the interaction with this j atom. */
309 velecsum = _mm_add_ps(velecsum,velec);
313 /* Update vectorial force */
314 fix0 = _mm_macc_ps(dx00,fscal,fix0);
315 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
316 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
318 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
319 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
320 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
322 /**************************
323 * CALCULATE INTERACTIONS *
324 **************************/
326 r01 = _mm_mul_ps(rsq01,rinv01);
328 /* EWALD ELECTROSTATICS */
330 /* Analytical PME correction */
331 zeta2 = _mm_mul_ps(beta2,rsq01);
332 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
333 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
334 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
335 felec = _mm_mul_ps(qq01,felec);
336 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
337 velec = _mm_nmacc_ps(pmecorrV,beta,rinv01);
338 velec = _mm_mul_ps(qq01,velec);
340 /* Update potential sum for this i atom from the interaction with this j atom. */
341 velecsum = _mm_add_ps(velecsum,velec);
345 /* Update vectorial force */
346 fix0 = _mm_macc_ps(dx01,fscal,fix0);
347 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
348 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
350 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
351 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
352 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
354 /**************************
355 * CALCULATE INTERACTIONS *
356 **************************/
358 r02 = _mm_mul_ps(rsq02,rinv02);
360 /* EWALD ELECTROSTATICS */
362 /* Analytical PME correction */
363 zeta2 = _mm_mul_ps(beta2,rsq02);
364 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
365 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
366 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
367 felec = _mm_mul_ps(qq02,felec);
368 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
369 velec = _mm_nmacc_ps(pmecorrV,beta,rinv02);
370 velec = _mm_mul_ps(qq02,velec);
372 /* Update potential sum for this i atom from the interaction with this j atom. */
373 velecsum = _mm_add_ps(velecsum,velec);
377 /* Update vectorial force */
378 fix0 = _mm_macc_ps(dx02,fscal,fix0);
379 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
380 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
382 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
383 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
384 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
386 /**************************
387 * CALCULATE INTERACTIONS *
388 **************************/
390 r10 = _mm_mul_ps(rsq10,rinv10);
392 /* EWALD ELECTROSTATICS */
394 /* Analytical PME correction */
395 zeta2 = _mm_mul_ps(beta2,rsq10);
396 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
397 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
398 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
399 felec = _mm_mul_ps(qq10,felec);
400 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
401 velec = _mm_nmacc_ps(pmecorrV,beta,rinv10);
402 velec = _mm_mul_ps(qq10,velec);
404 /* Update potential sum for this i atom from the interaction with this j atom. */
405 velecsum = _mm_add_ps(velecsum,velec);
409 /* Update vectorial force */
410 fix1 = _mm_macc_ps(dx10,fscal,fix1);
411 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
412 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
414 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
415 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
416 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
418 /**************************
419 * CALCULATE INTERACTIONS *
420 **************************/
422 r11 = _mm_mul_ps(rsq11,rinv11);
424 /* EWALD ELECTROSTATICS */
426 /* Analytical PME correction */
427 zeta2 = _mm_mul_ps(beta2,rsq11);
428 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
429 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
430 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
431 felec = _mm_mul_ps(qq11,felec);
432 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
433 velec = _mm_nmacc_ps(pmecorrV,beta,rinv11);
434 velec = _mm_mul_ps(qq11,velec);
436 /* Update potential sum for this i atom from the interaction with this j atom. */
437 velecsum = _mm_add_ps(velecsum,velec);
441 /* Update vectorial force */
442 fix1 = _mm_macc_ps(dx11,fscal,fix1);
443 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
444 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
446 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
447 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
448 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
450 /**************************
451 * CALCULATE INTERACTIONS *
452 **************************/
454 r12 = _mm_mul_ps(rsq12,rinv12);
456 /* EWALD ELECTROSTATICS */
458 /* Analytical PME correction */
459 zeta2 = _mm_mul_ps(beta2,rsq12);
460 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
461 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
462 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
463 felec = _mm_mul_ps(qq12,felec);
464 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
465 velec = _mm_nmacc_ps(pmecorrV,beta,rinv12);
466 velec = _mm_mul_ps(qq12,velec);
468 /* Update potential sum for this i atom from the interaction with this j atom. */
469 velecsum = _mm_add_ps(velecsum,velec);
473 /* Update vectorial force */
474 fix1 = _mm_macc_ps(dx12,fscal,fix1);
475 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
476 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
478 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
479 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
480 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
482 /**************************
483 * CALCULATE INTERACTIONS *
484 **************************/
486 r20 = _mm_mul_ps(rsq20,rinv20);
488 /* EWALD ELECTROSTATICS */
490 /* Analytical PME correction */
491 zeta2 = _mm_mul_ps(beta2,rsq20);
492 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
493 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
494 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
495 felec = _mm_mul_ps(qq20,felec);
496 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
497 velec = _mm_nmacc_ps(pmecorrV,beta,rinv20);
498 velec = _mm_mul_ps(qq20,velec);
500 /* Update potential sum for this i atom from the interaction with this j atom. */
501 velecsum = _mm_add_ps(velecsum,velec);
505 /* Update vectorial force */
506 fix2 = _mm_macc_ps(dx20,fscal,fix2);
507 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
508 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
510 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
511 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
512 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
514 /**************************
515 * CALCULATE INTERACTIONS *
516 **************************/
518 r21 = _mm_mul_ps(rsq21,rinv21);
520 /* EWALD ELECTROSTATICS */
522 /* Analytical PME correction */
523 zeta2 = _mm_mul_ps(beta2,rsq21);
524 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
525 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
526 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
527 felec = _mm_mul_ps(qq21,felec);
528 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
529 velec = _mm_nmacc_ps(pmecorrV,beta,rinv21);
530 velec = _mm_mul_ps(qq21,velec);
532 /* Update potential sum for this i atom from the interaction with this j atom. */
533 velecsum = _mm_add_ps(velecsum,velec);
537 /* Update vectorial force */
538 fix2 = _mm_macc_ps(dx21,fscal,fix2);
539 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
540 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
542 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
543 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
544 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
546 /**************************
547 * CALCULATE INTERACTIONS *
548 **************************/
550 r22 = _mm_mul_ps(rsq22,rinv22);
552 /* EWALD ELECTROSTATICS */
554 /* Analytical PME correction */
555 zeta2 = _mm_mul_ps(beta2,rsq22);
556 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
557 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
558 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
559 felec = _mm_mul_ps(qq22,felec);
560 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
561 velec = _mm_nmacc_ps(pmecorrV,beta,rinv22);
562 velec = _mm_mul_ps(qq22,velec);
564 /* Update potential sum for this i atom from the interaction with this j atom. */
565 velecsum = _mm_add_ps(velecsum,velec);
569 /* Update vectorial force */
570 fix2 = _mm_macc_ps(dx22,fscal,fix2);
571 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
572 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
574 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
575 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
576 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
578 fjptrA = f+j_coord_offsetA;
579 fjptrB = f+j_coord_offsetB;
580 fjptrC = f+j_coord_offsetC;
581 fjptrD = f+j_coord_offsetD;
583 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
584 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
586 /* Inner loop uses 261 flops */
592 /* Get j neighbor index, and coordinate index */
593 jnrlistA = jjnr[jidx];
594 jnrlistB = jjnr[jidx+1];
595 jnrlistC = jjnr[jidx+2];
596 jnrlistD = jjnr[jidx+3];
597 /* Sign of each element will be negative for non-real atoms.
598 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
599 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
601 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
602 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
603 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
604 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
605 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
606 j_coord_offsetA = DIM*jnrA;
607 j_coord_offsetB = DIM*jnrB;
608 j_coord_offsetC = DIM*jnrC;
609 j_coord_offsetD = DIM*jnrD;
611 /* load j atom coordinates */
612 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
613 x+j_coord_offsetC,x+j_coord_offsetD,
614 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
616 /* Calculate displacement vector */
617 dx00 = _mm_sub_ps(ix0,jx0);
618 dy00 = _mm_sub_ps(iy0,jy0);
619 dz00 = _mm_sub_ps(iz0,jz0);
620 dx01 = _mm_sub_ps(ix0,jx1);
621 dy01 = _mm_sub_ps(iy0,jy1);
622 dz01 = _mm_sub_ps(iz0,jz1);
623 dx02 = _mm_sub_ps(ix0,jx2);
624 dy02 = _mm_sub_ps(iy0,jy2);
625 dz02 = _mm_sub_ps(iz0,jz2);
626 dx10 = _mm_sub_ps(ix1,jx0);
627 dy10 = _mm_sub_ps(iy1,jy0);
628 dz10 = _mm_sub_ps(iz1,jz0);
629 dx11 = _mm_sub_ps(ix1,jx1);
630 dy11 = _mm_sub_ps(iy1,jy1);
631 dz11 = _mm_sub_ps(iz1,jz1);
632 dx12 = _mm_sub_ps(ix1,jx2);
633 dy12 = _mm_sub_ps(iy1,jy2);
634 dz12 = _mm_sub_ps(iz1,jz2);
635 dx20 = _mm_sub_ps(ix2,jx0);
636 dy20 = _mm_sub_ps(iy2,jy0);
637 dz20 = _mm_sub_ps(iz2,jz0);
638 dx21 = _mm_sub_ps(ix2,jx1);
639 dy21 = _mm_sub_ps(iy2,jy1);
640 dz21 = _mm_sub_ps(iz2,jz1);
641 dx22 = _mm_sub_ps(ix2,jx2);
642 dy22 = _mm_sub_ps(iy2,jy2);
643 dz22 = _mm_sub_ps(iz2,jz2);
645 /* Calculate squared distance and things based on it */
646 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
647 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
648 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
649 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
650 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
651 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
652 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
653 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
654 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
656 rinv00 = gmx_mm_invsqrt_ps(rsq00);
657 rinv01 = gmx_mm_invsqrt_ps(rsq01);
658 rinv02 = gmx_mm_invsqrt_ps(rsq02);
659 rinv10 = gmx_mm_invsqrt_ps(rsq10);
660 rinv11 = gmx_mm_invsqrt_ps(rsq11);
661 rinv12 = gmx_mm_invsqrt_ps(rsq12);
662 rinv20 = gmx_mm_invsqrt_ps(rsq20);
663 rinv21 = gmx_mm_invsqrt_ps(rsq21);
664 rinv22 = gmx_mm_invsqrt_ps(rsq22);
666 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
667 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
668 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
669 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
670 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
671 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
672 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
673 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
674 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
676 fjx0 = _mm_setzero_ps();
677 fjy0 = _mm_setzero_ps();
678 fjz0 = _mm_setzero_ps();
679 fjx1 = _mm_setzero_ps();
680 fjy1 = _mm_setzero_ps();
681 fjz1 = _mm_setzero_ps();
682 fjx2 = _mm_setzero_ps();
683 fjy2 = _mm_setzero_ps();
684 fjz2 = _mm_setzero_ps();
686 /**************************
687 * CALCULATE INTERACTIONS *
688 **************************/
690 r00 = _mm_mul_ps(rsq00,rinv00);
691 r00 = _mm_andnot_ps(dummy_mask,r00);
693 /* EWALD ELECTROSTATICS */
695 /* Analytical PME correction */
696 zeta2 = _mm_mul_ps(beta2,rsq00);
697 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
698 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
699 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
700 felec = _mm_mul_ps(qq00,felec);
701 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
702 velec = _mm_nmacc_ps(pmecorrV,beta,rinv00);
703 velec = _mm_mul_ps(qq00,velec);
705 /* Update potential sum for this i atom from the interaction with this j atom. */
706 velec = _mm_andnot_ps(dummy_mask,velec);
707 velecsum = _mm_add_ps(velecsum,velec);
711 fscal = _mm_andnot_ps(dummy_mask,fscal);
713 /* Update vectorial force */
714 fix0 = _mm_macc_ps(dx00,fscal,fix0);
715 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
716 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
718 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
719 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
720 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
722 /**************************
723 * CALCULATE INTERACTIONS *
724 **************************/
726 r01 = _mm_mul_ps(rsq01,rinv01);
727 r01 = _mm_andnot_ps(dummy_mask,r01);
729 /* EWALD ELECTROSTATICS */
731 /* Analytical PME correction */
732 zeta2 = _mm_mul_ps(beta2,rsq01);
733 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
734 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
735 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
736 felec = _mm_mul_ps(qq01,felec);
737 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
738 velec = _mm_nmacc_ps(pmecorrV,beta,rinv01);
739 velec = _mm_mul_ps(qq01,velec);
741 /* Update potential sum for this i atom from the interaction with this j atom. */
742 velec = _mm_andnot_ps(dummy_mask,velec);
743 velecsum = _mm_add_ps(velecsum,velec);
747 fscal = _mm_andnot_ps(dummy_mask,fscal);
749 /* Update vectorial force */
750 fix0 = _mm_macc_ps(dx01,fscal,fix0);
751 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
752 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
754 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
755 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
756 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
758 /**************************
759 * CALCULATE INTERACTIONS *
760 **************************/
762 r02 = _mm_mul_ps(rsq02,rinv02);
763 r02 = _mm_andnot_ps(dummy_mask,r02);
765 /* EWALD ELECTROSTATICS */
767 /* Analytical PME correction */
768 zeta2 = _mm_mul_ps(beta2,rsq02);
769 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
770 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
771 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
772 felec = _mm_mul_ps(qq02,felec);
773 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
774 velec = _mm_nmacc_ps(pmecorrV,beta,rinv02);
775 velec = _mm_mul_ps(qq02,velec);
777 /* Update potential sum for this i atom from the interaction with this j atom. */
778 velec = _mm_andnot_ps(dummy_mask,velec);
779 velecsum = _mm_add_ps(velecsum,velec);
783 fscal = _mm_andnot_ps(dummy_mask,fscal);
785 /* Update vectorial force */
786 fix0 = _mm_macc_ps(dx02,fscal,fix0);
787 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
788 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
790 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
791 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
792 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
794 /**************************
795 * CALCULATE INTERACTIONS *
796 **************************/
798 r10 = _mm_mul_ps(rsq10,rinv10);
799 r10 = _mm_andnot_ps(dummy_mask,r10);
801 /* EWALD ELECTROSTATICS */
803 /* Analytical PME correction */
804 zeta2 = _mm_mul_ps(beta2,rsq10);
805 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
806 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
807 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
808 felec = _mm_mul_ps(qq10,felec);
809 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
810 velec = _mm_nmacc_ps(pmecorrV,beta,rinv10);
811 velec = _mm_mul_ps(qq10,velec);
813 /* Update potential sum for this i atom from the interaction with this j atom. */
814 velec = _mm_andnot_ps(dummy_mask,velec);
815 velecsum = _mm_add_ps(velecsum,velec);
819 fscal = _mm_andnot_ps(dummy_mask,fscal);
821 /* Update vectorial force */
822 fix1 = _mm_macc_ps(dx10,fscal,fix1);
823 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
824 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
826 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
827 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
828 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
830 /**************************
831 * CALCULATE INTERACTIONS *
832 **************************/
834 r11 = _mm_mul_ps(rsq11,rinv11);
835 r11 = _mm_andnot_ps(dummy_mask,r11);
837 /* EWALD ELECTROSTATICS */
839 /* Analytical PME correction */
840 zeta2 = _mm_mul_ps(beta2,rsq11);
841 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
842 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
843 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
844 felec = _mm_mul_ps(qq11,felec);
845 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
846 velec = _mm_nmacc_ps(pmecorrV,beta,rinv11);
847 velec = _mm_mul_ps(qq11,velec);
849 /* Update potential sum for this i atom from the interaction with this j atom. */
850 velec = _mm_andnot_ps(dummy_mask,velec);
851 velecsum = _mm_add_ps(velecsum,velec);
855 fscal = _mm_andnot_ps(dummy_mask,fscal);
857 /* Update vectorial force */
858 fix1 = _mm_macc_ps(dx11,fscal,fix1);
859 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
860 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
862 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
863 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
864 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
866 /**************************
867 * CALCULATE INTERACTIONS *
868 **************************/
870 r12 = _mm_mul_ps(rsq12,rinv12);
871 r12 = _mm_andnot_ps(dummy_mask,r12);
873 /* EWALD ELECTROSTATICS */
875 /* Analytical PME correction */
876 zeta2 = _mm_mul_ps(beta2,rsq12);
877 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
878 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
879 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
880 felec = _mm_mul_ps(qq12,felec);
881 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
882 velec = _mm_nmacc_ps(pmecorrV,beta,rinv12);
883 velec = _mm_mul_ps(qq12,velec);
885 /* Update potential sum for this i atom from the interaction with this j atom. */
886 velec = _mm_andnot_ps(dummy_mask,velec);
887 velecsum = _mm_add_ps(velecsum,velec);
891 fscal = _mm_andnot_ps(dummy_mask,fscal);
893 /* Update vectorial force */
894 fix1 = _mm_macc_ps(dx12,fscal,fix1);
895 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
896 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
898 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
899 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
900 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
902 /**************************
903 * CALCULATE INTERACTIONS *
904 **************************/
906 r20 = _mm_mul_ps(rsq20,rinv20);
907 r20 = _mm_andnot_ps(dummy_mask,r20);
909 /* EWALD ELECTROSTATICS */
911 /* Analytical PME correction */
912 zeta2 = _mm_mul_ps(beta2,rsq20);
913 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
914 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
915 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
916 felec = _mm_mul_ps(qq20,felec);
917 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
918 velec = _mm_nmacc_ps(pmecorrV,beta,rinv20);
919 velec = _mm_mul_ps(qq20,velec);
921 /* Update potential sum for this i atom from the interaction with this j atom. */
922 velec = _mm_andnot_ps(dummy_mask,velec);
923 velecsum = _mm_add_ps(velecsum,velec);
927 fscal = _mm_andnot_ps(dummy_mask,fscal);
929 /* Update vectorial force */
930 fix2 = _mm_macc_ps(dx20,fscal,fix2);
931 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
932 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
934 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
935 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
936 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
938 /**************************
939 * CALCULATE INTERACTIONS *
940 **************************/
942 r21 = _mm_mul_ps(rsq21,rinv21);
943 r21 = _mm_andnot_ps(dummy_mask,r21);
945 /* EWALD ELECTROSTATICS */
947 /* Analytical PME correction */
948 zeta2 = _mm_mul_ps(beta2,rsq21);
949 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
950 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
951 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
952 felec = _mm_mul_ps(qq21,felec);
953 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
954 velec = _mm_nmacc_ps(pmecorrV,beta,rinv21);
955 velec = _mm_mul_ps(qq21,velec);
957 /* Update potential sum for this i atom from the interaction with this j atom. */
958 velec = _mm_andnot_ps(dummy_mask,velec);
959 velecsum = _mm_add_ps(velecsum,velec);
963 fscal = _mm_andnot_ps(dummy_mask,fscal);
965 /* Update vectorial force */
966 fix2 = _mm_macc_ps(dx21,fscal,fix2);
967 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
968 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
970 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
971 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
972 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
974 /**************************
975 * CALCULATE INTERACTIONS *
976 **************************/
978 r22 = _mm_mul_ps(rsq22,rinv22);
979 r22 = _mm_andnot_ps(dummy_mask,r22);
981 /* EWALD ELECTROSTATICS */
983 /* Analytical PME correction */
984 zeta2 = _mm_mul_ps(beta2,rsq22);
985 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
986 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
987 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
988 felec = _mm_mul_ps(qq22,felec);
989 pmecorrV = gmx_mm_pmecorrV_ps(zeta2);
990 velec = _mm_nmacc_ps(pmecorrV,beta,rinv22);
991 velec = _mm_mul_ps(qq22,velec);
993 /* Update potential sum for this i atom from the interaction with this j atom. */
994 velec = _mm_andnot_ps(dummy_mask,velec);
995 velecsum = _mm_add_ps(velecsum,velec);
999 fscal = _mm_andnot_ps(dummy_mask,fscal);
1001 /* Update vectorial force */
1002 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1003 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1004 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1006 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1007 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1008 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1010 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1011 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1012 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1013 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1015 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1016 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1018 /* Inner loop uses 270 flops */
1021 /* End of innermost loop */
1023 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1024 f+i_coord_offset,fshift+i_shift_offset);
1027 /* Update potential energies */
1028 gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1030 /* Increment number of inner iterations */
1031 inneriter += j_index_end - j_index_start;
1033 /* Outer loop uses 19 flops */
1036 /* Increment number of outer iterations */
1039 /* Update outer/inner flops */
1041 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_W3W3_VF,outeriter*19 + inneriter*270);
1044 * Gromacs nonbonded kernel: nb_kernel_ElecEw_VdwNone_GeomW3W3_F_avx_128_fma_single
1045 * Electrostatics interaction: Ewald
1046 * VdW interaction: None
1047 * Geometry: Water3-Water3
1048 * Calculate force/pot: Force
1051 nb_kernel_ElecEw_VdwNone_GeomW3W3_F_avx_128_fma_single
1052 (t_nblist * gmx_restrict nlist,
1053 rvec * gmx_restrict xx,
1054 rvec * gmx_restrict ff,
1055 t_forcerec * gmx_restrict fr,
1056 t_mdatoms * gmx_restrict mdatoms,
1057 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1058 t_nrnb * gmx_restrict nrnb)
1060 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1061 * just 0 for non-waters.
1062 * Suffixes A,B,C,D refer to j loop unrolling done with AVX_128, e.g. for the four different
1063 * jnr indices corresponding to data put in the four positions in the SIMD register.
1065 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1066 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1067 int jnrA,jnrB,jnrC,jnrD;
1068 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1069 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1070 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1071 real rcutoff_scalar;
1072 real *shiftvec,*fshift,*x,*f;
1073 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1074 real scratch[4*DIM];
1075 __m128 fscal,rcutoff,rcutoff2,jidxall;
1077 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1079 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1081 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1082 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1083 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1084 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1085 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1086 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1087 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1088 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1089 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1090 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1091 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1092 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1093 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1094 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1095 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1096 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1097 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
1100 __m128 ewtabscale,eweps,twoeweps,sh_ewald,ewrt,ewtabhalfspace,ewtabF,ewtabFn,ewtabD,ewtabV;
1101 __m128 beta,beta2,beta3,zeta2,pmecorrF,pmecorrV,rinv3;
1103 __m128 dummy_mask,cutoff_mask;
1104 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1105 __m128 one = _mm_set1_ps(1.0);
1106 __m128 two = _mm_set1_ps(2.0);
1112 jindex = nlist->jindex;
1114 shiftidx = nlist->shift;
1116 shiftvec = fr->shift_vec[0];
1117 fshift = fr->fshift[0];
1118 facel = _mm_set1_ps(fr->epsfac);
1119 charge = mdatoms->chargeA;
1121 sh_ewald = _mm_set1_ps(fr->ic->sh_ewald);
1122 beta = _mm_set1_ps(fr->ic->ewaldcoeff_q);
1123 beta2 = _mm_mul_ps(beta,beta);
1124 beta3 = _mm_mul_ps(beta,beta2);
1125 ewtab = fr->ic->tabq_coul_F;
1126 ewtabscale = _mm_set1_ps(fr->ic->tabq_scale);
1127 ewtabhalfspace = _mm_set1_ps(0.5/fr->ic->tabq_scale);
1129 /* Setup water-specific parameters */
1130 inr = nlist->iinr[0];
1131 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1132 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1133 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1135 jq0 = _mm_set1_ps(charge[inr+0]);
1136 jq1 = _mm_set1_ps(charge[inr+1]);
1137 jq2 = _mm_set1_ps(charge[inr+2]);
1138 qq00 = _mm_mul_ps(iq0,jq0);
1139 qq01 = _mm_mul_ps(iq0,jq1);
1140 qq02 = _mm_mul_ps(iq0,jq2);
1141 qq10 = _mm_mul_ps(iq1,jq0);
1142 qq11 = _mm_mul_ps(iq1,jq1);
1143 qq12 = _mm_mul_ps(iq1,jq2);
1144 qq20 = _mm_mul_ps(iq2,jq0);
1145 qq21 = _mm_mul_ps(iq2,jq1);
1146 qq22 = _mm_mul_ps(iq2,jq2);
1148 /* Avoid stupid compiler warnings */
1149 jnrA = jnrB = jnrC = jnrD = 0;
1150 j_coord_offsetA = 0;
1151 j_coord_offsetB = 0;
1152 j_coord_offsetC = 0;
1153 j_coord_offsetD = 0;
1158 for(iidx=0;iidx<4*DIM;iidx++)
1160 scratch[iidx] = 0.0;
1163 /* Start outer loop over neighborlists */
1164 for(iidx=0; iidx<nri; iidx++)
1166 /* Load shift vector for this list */
1167 i_shift_offset = DIM*shiftidx[iidx];
1169 /* Load limits for loop over neighbors */
1170 j_index_start = jindex[iidx];
1171 j_index_end = jindex[iidx+1];
1173 /* Get outer coordinate index */
1175 i_coord_offset = DIM*inr;
1177 /* Load i particle coords and add shift vector */
1178 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1179 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1181 fix0 = _mm_setzero_ps();
1182 fiy0 = _mm_setzero_ps();
1183 fiz0 = _mm_setzero_ps();
1184 fix1 = _mm_setzero_ps();
1185 fiy1 = _mm_setzero_ps();
1186 fiz1 = _mm_setzero_ps();
1187 fix2 = _mm_setzero_ps();
1188 fiy2 = _mm_setzero_ps();
1189 fiz2 = _mm_setzero_ps();
1191 /* Start inner kernel loop */
1192 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1195 /* Get j neighbor index, and coordinate index */
1197 jnrB = jjnr[jidx+1];
1198 jnrC = jjnr[jidx+2];
1199 jnrD = jjnr[jidx+3];
1200 j_coord_offsetA = DIM*jnrA;
1201 j_coord_offsetB = DIM*jnrB;
1202 j_coord_offsetC = DIM*jnrC;
1203 j_coord_offsetD = DIM*jnrD;
1205 /* load j atom coordinates */
1206 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1207 x+j_coord_offsetC,x+j_coord_offsetD,
1208 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1210 /* Calculate displacement vector */
1211 dx00 = _mm_sub_ps(ix0,jx0);
1212 dy00 = _mm_sub_ps(iy0,jy0);
1213 dz00 = _mm_sub_ps(iz0,jz0);
1214 dx01 = _mm_sub_ps(ix0,jx1);
1215 dy01 = _mm_sub_ps(iy0,jy1);
1216 dz01 = _mm_sub_ps(iz0,jz1);
1217 dx02 = _mm_sub_ps(ix0,jx2);
1218 dy02 = _mm_sub_ps(iy0,jy2);
1219 dz02 = _mm_sub_ps(iz0,jz2);
1220 dx10 = _mm_sub_ps(ix1,jx0);
1221 dy10 = _mm_sub_ps(iy1,jy0);
1222 dz10 = _mm_sub_ps(iz1,jz0);
1223 dx11 = _mm_sub_ps(ix1,jx1);
1224 dy11 = _mm_sub_ps(iy1,jy1);
1225 dz11 = _mm_sub_ps(iz1,jz1);
1226 dx12 = _mm_sub_ps(ix1,jx2);
1227 dy12 = _mm_sub_ps(iy1,jy2);
1228 dz12 = _mm_sub_ps(iz1,jz2);
1229 dx20 = _mm_sub_ps(ix2,jx0);
1230 dy20 = _mm_sub_ps(iy2,jy0);
1231 dz20 = _mm_sub_ps(iz2,jz0);
1232 dx21 = _mm_sub_ps(ix2,jx1);
1233 dy21 = _mm_sub_ps(iy2,jy1);
1234 dz21 = _mm_sub_ps(iz2,jz1);
1235 dx22 = _mm_sub_ps(ix2,jx2);
1236 dy22 = _mm_sub_ps(iy2,jy2);
1237 dz22 = _mm_sub_ps(iz2,jz2);
1239 /* Calculate squared distance and things based on it */
1240 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1241 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1242 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1243 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1244 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1245 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1246 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1247 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1248 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1250 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1251 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1252 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1253 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1254 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1255 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1256 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1257 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1258 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1260 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1261 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1262 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1263 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1264 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1265 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1266 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1267 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1268 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1270 fjx0 = _mm_setzero_ps();
1271 fjy0 = _mm_setzero_ps();
1272 fjz0 = _mm_setzero_ps();
1273 fjx1 = _mm_setzero_ps();
1274 fjy1 = _mm_setzero_ps();
1275 fjz1 = _mm_setzero_ps();
1276 fjx2 = _mm_setzero_ps();
1277 fjy2 = _mm_setzero_ps();
1278 fjz2 = _mm_setzero_ps();
1280 /**************************
1281 * CALCULATE INTERACTIONS *
1282 **************************/
1284 r00 = _mm_mul_ps(rsq00,rinv00);
1286 /* EWALD ELECTROSTATICS */
1288 /* Analytical PME correction */
1289 zeta2 = _mm_mul_ps(beta2,rsq00);
1290 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
1291 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1292 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1293 felec = _mm_mul_ps(qq00,felec);
1297 /* Update vectorial force */
1298 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1299 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1300 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1302 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1303 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1304 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1306 /**************************
1307 * CALCULATE INTERACTIONS *
1308 **************************/
1310 r01 = _mm_mul_ps(rsq01,rinv01);
1312 /* EWALD ELECTROSTATICS */
1314 /* Analytical PME correction */
1315 zeta2 = _mm_mul_ps(beta2,rsq01);
1316 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
1317 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1318 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1319 felec = _mm_mul_ps(qq01,felec);
1323 /* Update vectorial force */
1324 fix0 = _mm_macc_ps(dx01,fscal,fix0);
1325 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
1326 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
1328 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
1329 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
1330 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
1332 /**************************
1333 * CALCULATE INTERACTIONS *
1334 **************************/
1336 r02 = _mm_mul_ps(rsq02,rinv02);
1338 /* EWALD ELECTROSTATICS */
1340 /* Analytical PME correction */
1341 zeta2 = _mm_mul_ps(beta2,rsq02);
1342 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
1343 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1344 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1345 felec = _mm_mul_ps(qq02,felec);
1349 /* Update vectorial force */
1350 fix0 = _mm_macc_ps(dx02,fscal,fix0);
1351 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
1352 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
1354 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
1355 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
1356 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
1358 /**************************
1359 * CALCULATE INTERACTIONS *
1360 **************************/
1362 r10 = _mm_mul_ps(rsq10,rinv10);
1364 /* EWALD ELECTROSTATICS */
1366 /* Analytical PME correction */
1367 zeta2 = _mm_mul_ps(beta2,rsq10);
1368 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
1369 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1370 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1371 felec = _mm_mul_ps(qq10,felec);
1375 /* Update vectorial force */
1376 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1377 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1378 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1380 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1381 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1382 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1384 /**************************
1385 * CALCULATE INTERACTIONS *
1386 **************************/
1388 r11 = _mm_mul_ps(rsq11,rinv11);
1390 /* EWALD ELECTROSTATICS */
1392 /* Analytical PME correction */
1393 zeta2 = _mm_mul_ps(beta2,rsq11);
1394 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
1395 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1396 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1397 felec = _mm_mul_ps(qq11,felec);
1401 /* Update vectorial force */
1402 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1403 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1404 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1406 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1407 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1408 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1410 /**************************
1411 * CALCULATE INTERACTIONS *
1412 **************************/
1414 r12 = _mm_mul_ps(rsq12,rinv12);
1416 /* EWALD ELECTROSTATICS */
1418 /* Analytical PME correction */
1419 zeta2 = _mm_mul_ps(beta2,rsq12);
1420 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
1421 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1422 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1423 felec = _mm_mul_ps(qq12,felec);
1427 /* Update vectorial force */
1428 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1429 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1430 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1432 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1433 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1434 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1436 /**************************
1437 * CALCULATE INTERACTIONS *
1438 **************************/
1440 r20 = _mm_mul_ps(rsq20,rinv20);
1442 /* EWALD ELECTROSTATICS */
1444 /* Analytical PME correction */
1445 zeta2 = _mm_mul_ps(beta2,rsq20);
1446 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
1447 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1448 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1449 felec = _mm_mul_ps(qq20,felec);
1453 /* Update vectorial force */
1454 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1455 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1456 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1458 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1459 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1460 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1462 /**************************
1463 * CALCULATE INTERACTIONS *
1464 **************************/
1466 r21 = _mm_mul_ps(rsq21,rinv21);
1468 /* EWALD ELECTROSTATICS */
1470 /* Analytical PME correction */
1471 zeta2 = _mm_mul_ps(beta2,rsq21);
1472 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
1473 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1474 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1475 felec = _mm_mul_ps(qq21,felec);
1479 /* Update vectorial force */
1480 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1481 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1482 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1484 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1485 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1486 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1488 /**************************
1489 * CALCULATE INTERACTIONS *
1490 **************************/
1492 r22 = _mm_mul_ps(rsq22,rinv22);
1494 /* EWALD ELECTROSTATICS */
1496 /* Analytical PME correction */
1497 zeta2 = _mm_mul_ps(beta2,rsq22);
1498 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
1499 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1500 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1501 felec = _mm_mul_ps(qq22,felec);
1505 /* Update vectorial force */
1506 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1507 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1508 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1510 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1511 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1512 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1514 fjptrA = f+j_coord_offsetA;
1515 fjptrB = f+j_coord_offsetB;
1516 fjptrC = f+j_coord_offsetC;
1517 fjptrD = f+j_coord_offsetD;
1519 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1520 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1522 /* Inner loop uses 252 flops */
1525 if(jidx<j_index_end)
1528 /* Get j neighbor index, and coordinate index */
1529 jnrlistA = jjnr[jidx];
1530 jnrlistB = jjnr[jidx+1];
1531 jnrlistC = jjnr[jidx+2];
1532 jnrlistD = jjnr[jidx+3];
1533 /* Sign of each element will be negative for non-real atoms.
1534 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1535 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1537 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1538 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1539 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1540 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1541 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1542 j_coord_offsetA = DIM*jnrA;
1543 j_coord_offsetB = DIM*jnrB;
1544 j_coord_offsetC = DIM*jnrC;
1545 j_coord_offsetD = DIM*jnrD;
1547 /* load j atom coordinates */
1548 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1549 x+j_coord_offsetC,x+j_coord_offsetD,
1550 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1552 /* Calculate displacement vector */
1553 dx00 = _mm_sub_ps(ix0,jx0);
1554 dy00 = _mm_sub_ps(iy0,jy0);
1555 dz00 = _mm_sub_ps(iz0,jz0);
1556 dx01 = _mm_sub_ps(ix0,jx1);
1557 dy01 = _mm_sub_ps(iy0,jy1);
1558 dz01 = _mm_sub_ps(iz0,jz1);
1559 dx02 = _mm_sub_ps(ix0,jx2);
1560 dy02 = _mm_sub_ps(iy0,jy2);
1561 dz02 = _mm_sub_ps(iz0,jz2);
1562 dx10 = _mm_sub_ps(ix1,jx0);
1563 dy10 = _mm_sub_ps(iy1,jy0);
1564 dz10 = _mm_sub_ps(iz1,jz0);
1565 dx11 = _mm_sub_ps(ix1,jx1);
1566 dy11 = _mm_sub_ps(iy1,jy1);
1567 dz11 = _mm_sub_ps(iz1,jz1);
1568 dx12 = _mm_sub_ps(ix1,jx2);
1569 dy12 = _mm_sub_ps(iy1,jy2);
1570 dz12 = _mm_sub_ps(iz1,jz2);
1571 dx20 = _mm_sub_ps(ix2,jx0);
1572 dy20 = _mm_sub_ps(iy2,jy0);
1573 dz20 = _mm_sub_ps(iz2,jz0);
1574 dx21 = _mm_sub_ps(ix2,jx1);
1575 dy21 = _mm_sub_ps(iy2,jy1);
1576 dz21 = _mm_sub_ps(iz2,jz1);
1577 dx22 = _mm_sub_ps(ix2,jx2);
1578 dy22 = _mm_sub_ps(iy2,jy2);
1579 dz22 = _mm_sub_ps(iz2,jz2);
1581 /* Calculate squared distance and things based on it */
1582 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1583 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1584 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1585 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1586 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1587 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1588 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1589 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1590 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1592 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1593 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1594 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1595 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1596 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1597 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1598 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1599 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1600 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1602 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1603 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1604 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1605 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1606 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1607 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1608 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1609 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1610 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1612 fjx0 = _mm_setzero_ps();
1613 fjy0 = _mm_setzero_ps();
1614 fjz0 = _mm_setzero_ps();
1615 fjx1 = _mm_setzero_ps();
1616 fjy1 = _mm_setzero_ps();
1617 fjz1 = _mm_setzero_ps();
1618 fjx2 = _mm_setzero_ps();
1619 fjy2 = _mm_setzero_ps();
1620 fjz2 = _mm_setzero_ps();
1622 /**************************
1623 * CALCULATE INTERACTIONS *
1624 **************************/
1626 r00 = _mm_mul_ps(rsq00,rinv00);
1627 r00 = _mm_andnot_ps(dummy_mask,r00);
1629 /* EWALD ELECTROSTATICS */
1631 /* Analytical PME correction */
1632 zeta2 = _mm_mul_ps(beta2,rsq00);
1633 rinv3 = _mm_mul_ps(rinvsq00,rinv00);
1634 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1635 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1636 felec = _mm_mul_ps(qq00,felec);
1640 fscal = _mm_andnot_ps(dummy_mask,fscal);
1642 /* Update vectorial force */
1643 fix0 = _mm_macc_ps(dx00,fscal,fix0);
1644 fiy0 = _mm_macc_ps(dy00,fscal,fiy0);
1645 fiz0 = _mm_macc_ps(dz00,fscal,fiz0);
1647 fjx0 = _mm_macc_ps(dx00,fscal,fjx0);
1648 fjy0 = _mm_macc_ps(dy00,fscal,fjy0);
1649 fjz0 = _mm_macc_ps(dz00,fscal,fjz0);
1651 /**************************
1652 * CALCULATE INTERACTIONS *
1653 **************************/
1655 r01 = _mm_mul_ps(rsq01,rinv01);
1656 r01 = _mm_andnot_ps(dummy_mask,r01);
1658 /* EWALD ELECTROSTATICS */
1660 /* Analytical PME correction */
1661 zeta2 = _mm_mul_ps(beta2,rsq01);
1662 rinv3 = _mm_mul_ps(rinvsq01,rinv01);
1663 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1664 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1665 felec = _mm_mul_ps(qq01,felec);
1669 fscal = _mm_andnot_ps(dummy_mask,fscal);
1671 /* Update vectorial force */
1672 fix0 = _mm_macc_ps(dx01,fscal,fix0);
1673 fiy0 = _mm_macc_ps(dy01,fscal,fiy0);
1674 fiz0 = _mm_macc_ps(dz01,fscal,fiz0);
1676 fjx1 = _mm_macc_ps(dx01,fscal,fjx1);
1677 fjy1 = _mm_macc_ps(dy01,fscal,fjy1);
1678 fjz1 = _mm_macc_ps(dz01,fscal,fjz1);
1680 /**************************
1681 * CALCULATE INTERACTIONS *
1682 **************************/
1684 r02 = _mm_mul_ps(rsq02,rinv02);
1685 r02 = _mm_andnot_ps(dummy_mask,r02);
1687 /* EWALD ELECTROSTATICS */
1689 /* Analytical PME correction */
1690 zeta2 = _mm_mul_ps(beta2,rsq02);
1691 rinv3 = _mm_mul_ps(rinvsq02,rinv02);
1692 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1693 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1694 felec = _mm_mul_ps(qq02,felec);
1698 fscal = _mm_andnot_ps(dummy_mask,fscal);
1700 /* Update vectorial force */
1701 fix0 = _mm_macc_ps(dx02,fscal,fix0);
1702 fiy0 = _mm_macc_ps(dy02,fscal,fiy0);
1703 fiz0 = _mm_macc_ps(dz02,fscal,fiz0);
1705 fjx2 = _mm_macc_ps(dx02,fscal,fjx2);
1706 fjy2 = _mm_macc_ps(dy02,fscal,fjy2);
1707 fjz2 = _mm_macc_ps(dz02,fscal,fjz2);
1709 /**************************
1710 * CALCULATE INTERACTIONS *
1711 **************************/
1713 r10 = _mm_mul_ps(rsq10,rinv10);
1714 r10 = _mm_andnot_ps(dummy_mask,r10);
1716 /* EWALD ELECTROSTATICS */
1718 /* Analytical PME correction */
1719 zeta2 = _mm_mul_ps(beta2,rsq10);
1720 rinv3 = _mm_mul_ps(rinvsq10,rinv10);
1721 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1722 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1723 felec = _mm_mul_ps(qq10,felec);
1727 fscal = _mm_andnot_ps(dummy_mask,fscal);
1729 /* Update vectorial force */
1730 fix1 = _mm_macc_ps(dx10,fscal,fix1);
1731 fiy1 = _mm_macc_ps(dy10,fscal,fiy1);
1732 fiz1 = _mm_macc_ps(dz10,fscal,fiz1);
1734 fjx0 = _mm_macc_ps(dx10,fscal,fjx0);
1735 fjy0 = _mm_macc_ps(dy10,fscal,fjy0);
1736 fjz0 = _mm_macc_ps(dz10,fscal,fjz0);
1738 /**************************
1739 * CALCULATE INTERACTIONS *
1740 **************************/
1742 r11 = _mm_mul_ps(rsq11,rinv11);
1743 r11 = _mm_andnot_ps(dummy_mask,r11);
1745 /* EWALD ELECTROSTATICS */
1747 /* Analytical PME correction */
1748 zeta2 = _mm_mul_ps(beta2,rsq11);
1749 rinv3 = _mm_mul_ps(rinvsq11,rinv11);
1750 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1751 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1752 felec = _mm_mul_ps(qq11,felec);
1756 fscal = _mm_andnot_ps(dummy_mask,fscal);
1758 /* Update vectorial force */
1759 fix1 = _mm_macc_ps(dx11,fscal,fix1);
1760 fiy1 = _mm_macc_ps(dy11,fscal,fiy1);
1761 fiz1 = _mm_macc_ps(dz11,fscal,fiz1);
1763 fjx1 = _mm_macc_ps(dx11,fscal,fjx1);
1764 fjy1 = _mm_macc_ps(dy11,fscal,fjy1);
1765 fjz1 = _mm_macc_ps(dz11,fscal,fjz1);
1767 /**************************
1768 * CALCULATE INTERACTIONS *
1769 **************************/
1771 r12 = _mm_mul_ps(rsq12,rinv12);
1772 r12 = _mm_andnot_ps(dummy_mask,r12);
1774 /* EWALD ELECTROSTATICS */
1776 /* Analytical PME correction */
1777 zeta2 = _mm_mul_ps(beta2,rsq12);
1778 rinv3 = _mm_mul_ps(rinvsq12,rinv12);
1779 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1780 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1781 felec = _mm_mul_ps(qq12,felec);
1785 fscal = _mm_andnot_ps(dummy_mask,fscal);
1787 /* Update vectorial force */
1788 fix1 = _mm_macc_ps(dx12,fscal,fix1);
1789 fiy1 = _mm_macc_ps(dy12,fscal,fiy1);
1790 fiz1 = _mm_macc_ps(dz12,fscal,fiz1);
1792 fjx2 = _mm_macc_ps(dx12,fscal,fjx2);
1793 fjy2 = _mm_macc_ps(dy12,fscal,fjy2);
1794 fjz2 = _mm_macc_ps(dz12,fscal,fjz2);
1796 /**************************
1797 * CALCULATE INTERACTIONS *
1798 **************************/
1800 r20 = _mm_mul_ps(rsq20,rinv20);
1801 r20 = _mm_andnot_ps(dummy_mask,r20);
1803 /* EWALD ELECTROSTATICS */
1805 /* Analytical PME correction */
1806 zeta2 = _mm_mul_ps(beta2,rsq20);
1807 rinv3 = _mm_mul_ps(rinvsq20,rinv20);
1808 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1809 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1810 felec = _mm_mul_ps(qq20,felec);
1814 fscal = _mm_andnot_ps(dummy_mask,fscal);
1816 /* Update vectorial force */
1817 fix2 = _mm_macc_ps(dx20,fscal,fix2);
1818 fiy2 = _mm_macc_ps(dy20,fscal,fiy2);
1819 fiz2 = _mm_macc_ps(dz20,fscal,fiz2);
1821 fjx0 = _mm_macc_ps(dx20,fscal,fjx0);
1822 fjy0 = _mm_macc_ps(dy20,fscal,fjy0);
1823 fjz0 = _mm_macc_ps(dz20,fscal,fjz0);
1825 /**************************
1826 * CALCULATE INTERACTIONS *
1827 **************************/
1829 r21 = _mm_mul_ps(rsq21,rinv21);
1830 r21 = _mm_andnot_ps(dummy_mask,r21);
1832 /* EWALD ELECTROSTATICS */
1834 /* Analytical PME correction */
1835 zeta2 = _mm_mul_ps(beta2,rsq21);
1836 rinv3 = _mm_mul_ps(rinvsq21,rinv21);
1837 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1838 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1839 felec = _mm_mul_ps(qq21,felec);
1843 fscal = _mm_andnot_ps(dummy_mask,fscal);
1845 /* Update vectorial force */
1846 fix2 = _mm_macc_ps(dx21,fscal,fix2);
1847 fiy2 = _mm_macc_ps(dy21,fscal,fiy2);
1848 fiz2 = _mm_macc_ps(dz21,fscal,fiz2);
1850 fjx1 = _mm_macc_ps(dx21,fscal,fjx1);
1851 fjy1 = _mm_macc_ps(dy21,fscal,fjy1);
1852 fjz1 = _mm_macc_ps(dz21,fscal,fjz1);
1854 /**************************
1855 * CALCULATE INTERACTIONS *
1856 **************************/
1858 r22 = _mm_mul_ps(rsq22,rinv22);
1859 r22 = _mm_andnot_ps(dummy_mask,r22);
1861 /* EWALD ELECTROSTATICS */
1863 /* Analytical PME correction */
1864 zeta2 = _mm_mul_ps(beta2,rsq22);
1865 rinv3 = _mm_mul_ps(rinvsq22,rinv22);
1866 pmecorrF = gmx_mm_pmecorrF_ps(zeta2);
1867 felec = _mm_macc_ps(pmecorrF,beta3,rinv3);
1868 felec = _mm_mul_ps(qq22,felec);
1872 fscal = _mm_andnot_ps(dummy_mask,fscal);
1874 /* Update vectorial force */
1875 fix2 = _mm_macc_ps(dx22,fscal,fix2);
1876 fiy2 = _mm_macc_ps(dy22,fscal,fiy2);
1877 fiz2 = _mm_macc_ps(dz22,fscal,fiz2);
1879 fjx2 = _mm_macc_ps(dx22,fscal,fjx2);
1880 fjy2 = _mm_macc_ps(dy22,fscal,fjy2);
1881 fjz2 = _mm_macc_ps(dz22,fscal,fjz2);
1883 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1884 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1885 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1886 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1888 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1889 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1891 /* Inner loop uses 261 flops */
1894 /* End of innermost loop */
1896 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1897 f+i_coord_offset,fshift+i_shift_offset);
1899 /* Increment number of inner iterations */
1900 inneriter += j_index_end - j_index_start;
1902 /* Outer loop uses 18 flops */
1905 /* Increment number of outer iterations */
1908 /* Update outer/inner flops */
1910 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_W3W3_F,outeriter*18 + inneriter*261);