2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS sse2_single kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/legacyheaders/types/simple.h"
46 #include "gromacs/math/vec.h"
47 #include "gromacs/legacyheaders/nrnb.h"
49 #include "gromacs/simd/math_x86_sse2_single.h"
50 #include "kernelutil_x86_sse2_single.h"
53 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_VF_sse2_single
54 * Electrostatics interaction: Coulomb
55 * VdW interaction: CubicSplineTable
56 * Geometry: Water3-Water3
57 * Calculate force/pot: PotentialAndForce
60 nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_VF_sse2_single
61 (t_nblist * gmx_restrict nlist,
62 rvec * gmx_restrict xx,
63 rvec * gmx_restrict ff,
64 t_forcerec * gmx_restrict fr,
65 t_mdatoms * gmx_restrict mdatoms,
66 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
67 t_nrnb * gmx_restrict nrnb)
69 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
70 * just 0 for non-waters.
71 * Suffixes A,B,C,D refer to j loop unrolling done with SSE, e.g. for the four different
72 * jnr indices corresponding to data put in the four positions in the SIMD register.
74 int i_shift_offset,i_coord_offset,outeriter,inneriter;
75 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
76 int jnrA,jnrB,jnrC,jnrD;
77 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
78 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
79 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
81 real *shiftvec,*fshift,*x,*f;
82 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
84 __m128 tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
86 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
88 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
90 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
91 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
92 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
93 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
94 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
95 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
96 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
97 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
98 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
99 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
100 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
101 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
102 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
103 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
104 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
105 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
106 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
109 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
112 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
113 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
115 __m128i ifour = _mm_set1_epi32(4);
116 __m128 rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
118 __m128 dummy_mask,cutoff_mask;
119 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
120 __m128 one = _mm_set1_ps(1.0);
121 __m128 two = _mm_set1_ps(2.0);
127 jindex = nlist->jindex;
129 shiftidx = nlist->shift;
131 shiftvec = fr->shift_vec[0];
132 fshift = fr->fshift[0];
133 facel = _mm_set1_ps(fr->epsfac);
134 charge = mdatoms->chargeA;
135 nvdwtype = fr->ntype;
137 vdwtype = mdatoms->typeA;
139 vftab = kernel_data->table_vdw->data;
140 vftabscale = _mm_set1_ps(kernel_data->table_vdw->scale);
142 /* Setup water-specific parameters */
143 inr = nlist->iinr[0];
144 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
145 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
146 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
147 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
149 jq0 = _mm_set1_ps(charge[inr+0]);
150 jq1 = _mm_set1_ps(charge[inr+1]);
151 jq2 = _mm_set1_ps(charge[inr+2]);
152 vdwjidx0A = 2*vdwtype[inr+0];
153 qq00 = _mm_mul_ps(iq0,jq0);
154 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
155 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
156 qq01 = _mm_mul_ps(iq0,jq1);
157 qq02 = _mm_mul_ps(iq0,jq2);
158 qq10 = _mm_mul_ps(iq1,jq0);
159 qq11 = _mm_mul_ps(iq1,jq1);
160 qq12 = _mm_mul_ps(iq1,jq2);
161 qq20 = _mm_mul_ps(iq2,jq0);
162 qq21 = _mm_mul_ps(iq2,jq1);
163 qq22 = _mm_mul_ps(iq2,jq2);
165 /* Avoid stupid compiler warnings */
166 jnrA = jnrB = jnrC = jnrD = 0;
175 for(iidx=0;iidx<4*DIM;iidx++)
180 /* Start outer loop over neighborlists */
181 for(iidx=0; iidx<nri; iidx++)
183 /* Load shift vector for this list */
184 i_shift_offset = DIM*shiftidx[iidx];
186 /* Load limits for loop over neighbors */
187 j_index_start = jindex[iidx];
188 j_index_end = jindex[iidx+1];
190 /* Get outer coordinate index */
192 i_coord_offset = DIM*inr;
194 /* Load i particle coords and add shift vector */
195 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
196 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
198 fix0 = _mm_setzero_ps();
199 fiy0 = _mm_setzero_ps();
200 fiz0 = _mm_setzero_ps();
201 fix1 = _mm_setzero_ps();
202 fiy1 = _mm_setzero_ps();
203 fiz1 = _mm_setzero_ps();
204 fix2 = _mm_setzero_ps();
205 fiy2 = _mm_setzero_ps();
206 fiz2 = _mm_setzero_ps();
208 /* Reset potential sums */
209 velecsum = _mm_setzero_ps();
210 vvdwsum = _mm_setzero_ps();
212 /* Start inner kernel loop */
213 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
216 /* Get j neighbor index, and coordinate index */
221 j_coord_offsetA = DIM*jnrA;
222 j_coord_offsetB = DIM*jnrB;
223 j_coord_offsetC = DIM*jnrC;
224 j_coord_offsetD = DIM*jnrD;
226 /* load j atom coordinates */
227 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
228 x+j_coord_offsetC,x+j_coord_offsetD,
229 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
231 /* Calculate displacement vector */
232 dx00 = _mm_sub_ps(ix0,jx0);
233 dy00 = _mm_sub_ps(iy0,jy0);
234 dz00 = _mm_sub_ps(iz0,jz0);
235 dx01 = _mm_sub_ps(ix0,jx1);
236 dy01 = _mm_sub_ps(iy0,jy1);
237 dz01 = _mm_sub_ps(iz0,jz1);
238 dx02 = _mm_sub_ps(ix0,jx2);
239 dy02 = _mm_sub_ps(iy0,jy2);
240 dz02 = _mm_sub_ps(iz0,jz2);
241 dx10 = _mm_sub_ps(ix1,jx0);
242 dy10 = _mm_sub_ps(iy1,jy0);
243 dz10 = _mm_sub_ps(iz1,jz0);
244 dx11 = _mm_sub_ps(ix1,jx1);
245 dy11 = _mm_sub_ps(iy1,jy1);
246 dz11 = _mm_sub_ps(iz1,jz1);
247 dx12 = _mm_sub_ps(ix1,jx2);
248 dy12 = _mm_sub_ps(iy1,jy2);
249 dz12 = _mm_sub_ps(iz1,jz2);
250 dx20 = _mm_sub_ps(ix2,jx0);
251 dy20 = _mm_sub_ps(iy2,jy0);
252 dz20 = _mm_sub_ps(iz2,jz0);
253 dx21 = _mm_sub_ps(ix2,jx1);
254 dy21 = _mm_sub_ps(iy2,jy1);
255 dz21 = _mm_sub_ps(iz2,jz1);
256 dx22 = _mm_sub_ps(ix2,jx2);
257 dy22 = _mm_sub_ps(iy2,jy2);
258 dz22 = _mm_sub_ps(iz2,jz2);
260 /* Calculate squared distance and things based on it */
261 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
262 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
263 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
264 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
265 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
266 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
267 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
268 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
269 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
271 rinv00 = gmx_mm_invsqrt_ps(rsq00);
272 rinv01 = gmx_mm_invsqrt_ps(rsq01);
273 rinv02 = gmx_mm_invsqrt_ps(rsq02);
274 rinv10 = gmx_mm_invsqrt_ps(rsq10);
275 rinv11 = gmx_mm_invsqrt_ps(rsq11);
276 rinv12 = gmx_mm_invsqrt_ps(rsq12);
277 rinv20 = gmx_mm_invsqrt_ps(rsq20);
278 rinv21 = gmx_mm_invsqrt_ps(rsq21);
279 rinv22 = gmx_mm_invsqrt_ps(rsq22);
281 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
282 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
283 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
284 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
285 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
286 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
287 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
288 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
289 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
291 fjx0 = _mm_setzero_ps();
292 fjy0 = _mm_setzero_ps();
293 fjz0 = _mm_setzero_ps();
294 fjx1 = _mm_setzero_ps();
295 fjy1 = _mm_setzero_ps();
296 fjz1 = _mm_setzero_ps();
297 fjx2 = _mm_setzero_ps();
298 fjy2 = _mm_setzero_ps();
299 fjz2 = _mm_setzero_ps();
301 /**************************
302 * CALCULATE INTERACTIONS *
303 **************************/
305 r00 = _mm_mul_ps(rsq00,rinv00);
307 /* Calculate table index by multiplying r with table scale and truncate to integer */
308 rt = _mm_mul_ps(r00,vftabscale);
309 vfitab = _mm_cvttps_epi32(rt);
310 vfeps = _mm_sub_ps(rt,_mm_cvtepi32_ps(vfitab));
311 vfitab = _mm_slli_epi32(vfitab,3);
313 /* COULOMB ELECTROSTATICS */
314 velec = _mm_mul_ps(qq00,rinv00);
315 felec = _mm_mul_ps(velec,rinvsq00);
317 /* CUBIC SPLINE TABLE DISPERSION */
318 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
319 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
320 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
321 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
322 _MM_TRANSPOSE4_PS(Y,F,G,H);
323 Heps = _mm_mul_ps(vfeps,H);
324 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
325 VV = _mm_add_ps(Y,_mm_mul_ps(vfeps,Fp));
326 vvdw6 = _mm_mul_ps(c6_00,VV);
327 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
328 fvdw6 = _mm_mul_ps(c6_00,FF);
330 /* CUBIC SPLINE TABLE REPULSION */
331 vfitab = _mm_add_epi32(vfitab,ifour);
332 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
333 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
334 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
335 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
336 _MM_TRANSPOSE4_PS(Y,F,G,H);
337 Heps = _mm_mul_ps(vfeps,H);
338 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
339 VV = _mm_add_ps(Y,_mm_mul_ps(vfeps,Fp));
340 vvdw12 = _mm_mul_ps(c12_00,VV);
341 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
342 fvdw12 = _mm_mul_ps(c12_00,FF);
343 vvdw = _mm_add_ps(vvdw12,vvdw6);
344 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
346 /* Update potential sum for this i atom from the interaction with this j atom. */
347 velecsum = _mm_add_ps(velecsum,velec);
348 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
350 fscal = _mm_add_ps(felec,fvdw);
352 /* Calculate temporary vectorial force */
353 tx = _mm_mul_ps(fscal,dx00);
354 ty = _mm_mul_ps(fscal,dy00);
355 tz = _mm_mul_ps(fscal,dz00);
357 /* Update vectorial force */
358 fix0 = _mm_add_ps(fix0,tx);
359 fiy0 = _mm_add_ps(fiy0,ty);
360 fiz0 = _mm_add_ps(fiz0,tz);
362 fjx0 = _mm_add_ps(fjx0,tx);
363 fjy0 = _mm_add_ps(fjy0,ty);
364 fjz0 = _mm_add_ps(fjz0,tz);
366 /**************************
367 * CALCULATE INTERACTIONS *
368 **************************/
370 /* COULOMB ELECTROSTATICS */
371 velec = _mm_mul_ps(qq01,rinv01);
372 felec = _mm_mul_ps(velec,rinvsq01);
374 /* Update potential sum for this i atom from the interaction with this j atom. */
375 velecsum = _mm_add_ps(velecsum,velec);
379 /* Calculate temporary vectorial force */
380 tx = _mm_mul_ps(fscal,dx01);
381 ty = _mm_mul_ps(fscal,dy01);
382 tz = _mm_mul_ps(fscal,dz01);
384 /* Update vectorial force */
385 fix0 = _mm_add_ps(fix0,tx);
386 fiy0 = _mm_add_ps(fiy0,ty);
387 fiz0 = _mm_add_ps(fiz0,tz);
389 fjx1 = _mm_add_ps(fjx1,tx);
390 fjy1 = _mm_add_ps(fjy1,ty);
391 fjz1 = _mm_add_ps(fjz1,tz);
393 /**************************
394 * CALCULATE INTERACTIONS *
395 **************************/
397 /* COULOMB ELECTROSTATICS */
398 velec = _mm_mul_ps(qq02,rinv02);
399 felec = _mm_mul_ps(velec,rinvsq02);
401 /* Update potential sum for this i atom from the interaction with this j atom. */
402 velecsum = _mm_add_ps(velecsum,velec);
406 /* Calculate temporary vectorial force */
407 tx = _mm_mul_ps(fscal,dx02);
408 ty = _mm_mul_ps(fscal,dy02);
409 tz = _mm_mul_ps(fscal,dz02);
411 /* Update vectorial force */
412 fix0 = _mm_add_ps(fix0,tx);
413 fiy0 = _mm_add_ps(fiy0,ty);
414 fiz0 = _mm_add_ps(fiz0,tz);
416 fjx2 = _mm_add_ps(fjx2,tx);
417 fjy2 = _mm_add_ps(fjy2,ty);
418 fjz2 = _mm_add_ps(fjz2,tz);
420 /**************************
421 * CALCULATE INTERACTIONS *
422 **************************/
424 /* COULOMB ELECTROSTATICS */
425 velec = _mm_mul_ps(qq10,rinv10);
426 felec = _mm_mul_ps(velec,rinvsq10);
428 /* Update potential sum for this i atom from the interaction with this j atom. */
429 velecsum = _mm_add_ps(velecsum,velec);
433 /* Calculate temporary vectorial force */
434 tx = _mm_mul_ps(fscal,dx10);
435 ty = _mm_mul_ps(fscal,dy10);
436 tz = _mm_mul_ps(fscal,dz10);
438 /* Update vectorial force */
439 fix1 = _mm_add_ps(fix1,tx);
440 fiy1 = _mm_add_ps(fiy1,ty);
441 fiz1 = _mm_add_ps(fiz1,tz);
443 fjx0 = _mm_add_ps(fjx0,tx);
444 fjy0 = _mm_add_ps(fjy0,ty);
445 fjz0 = _mm_add_ps(fjz0,tz);
447 /**************************
448 * CALCULATE INTERACTIONS *
449 **************************/
451 /* COULOMB ELECTROSTATICS */
452 velec = _mm_mul_ps(qq11,rinv11);
453 felec = _mm_mul_ps(velec,rinvsq11);
455 /* Update potential sum for this i atom from the interaction with this j atom. */
456 velecsum = _mm_add_ps(velecsum,velec);
460 /* Calculate temporary vectorial force */
461 tx = _mm_mul_ps(fscal,dx11);
462 ty = _mm_mul_ps(fscal,dy11);
463 tz = _mm_mul_ps(fscal,dz11);
465 /* Update vectorial force */
466 fix1 = _mm_add_ps(fix1,tx);
467 fiy1 = _mm_add_ps(fiy1,ty);
468 fiz1 = _mm_add_ps(fiz1,tz);
470 fjx1 = _mm_add_ps(fjx1,tx);
471 fjy1 = _mm_add_ps(fjy1,ty);
472 fjz1 = _mm_add_ps(fjz1,tz);
474 /**************************
475 * CALCULATE INTERACTIONS *
476 **************************/
478 /* COULOMB ELECTROSTATICS */
479 velec = _mm_mul_ps(qq12,rinv12);
480 felec = _mm_mul_ps(velec,rinvsq12);
482 /* Update potential sum for this i atom from the interaction with this j atom. */
483 velecsum = _mm_add_ps(velecsum,velec);
487 /* Calculate temporary vectorial force */
488 tx = _mm_mul_ps(fscal,dx12);
489 ty = _mm_mul_ps(fscal,dy12);
490 tz = _mm_mul_ps(fscal,dz12);
492 /* Update vectorial force */
493 fix1 = _mm_add_ps(fix1,tx);
494 fiy1 = _mm_add_ps(fiy1,ty);
495 fiz1 = _mm_add_ps(fiz1,tz);
497 fjx2 = _mm_add_ps(fjx2,tx);
498 fjy2 = _mm_add_ps(fjy2,ty);
499 fjz2 = _mm_add_ps(fjz2,tz);
501 /**************************
502 * CALCULATE INTERACTIONS *
503 **************************/
505 /* COULOMB ELECTROSTATICS */
506 velec = _mm_mul_ps(qq20,rinv20);
507 felec = _mm_mul_ps(velec,rinvsq20);
509 /* Update potential sum for this i atom from the interaction with this j atom. */
510 velecsum = _mm_add_ps(velecsum,velec);
514 /* Calculate temporary vectorial force */
515 tx = _mm_mul_ps(fscal,dx20);
516 ty = _mm_mul_ps(fscal,dy20);
517 tz = _mm_mul_ps(fscal,dz20);
519 /* Update vectorial force */
520 fix2 = _mm_add_ps(fix2,tx);
521 fiy2 = _mm_add_ps(fiy2,ty);
522 fiz2 = _mm_add_ps(fiz2,tz);
524 fjx0 = _mm_add_ps(fjx0,tx);
525 fjy0 = _mm_add_ps(fjy0,ty);
526 fjz0 = _mm_add_ps(fjz0,tz);
528 /**************************
529 * CALCULATE INTERACTIONS *
530 **************************/
532 /* COULOMB ELECTROSTATICS */
533 velec = _mm_mul_ps(qq21,rinv21);
534 felec = _mm_mul_ps(velec,rinvsq21);
536 /* Update potential sum for this i atom from the interaction with this j atom. */
537 velecsum = _mm_add_ps(velecsum,velec);
541 /* Calculate temporary vectorial force */
542 tx = _mm_mul_ps(fscal,dx21);
543 ty = _mm_mul_ps(fscal,dy21);
544 tz = _mm_mul_ps(fscal,dz21);
546 /* Update vectorial force */
547 fix2 = _mm_add_ps(fix2,tx);
548 fiy2 = _mm_add_ps(fiy2,ty);
549 fiz2 = _mm_add_ps(fiz2,tz);
551 fjx1 = _mm_add_ps(fjx1,tx);
552 fjy1 = _mm_add_ps(fjy1,ty);
553 fjz1 = _mm_add_ps(fjz1,tz);
555 /**************************
556 * CALCULATE INTERACTIONS *
557 **************************/
559 /* COULOMB ELECTROSTATICS */
560 velec = _mm_mul_ps(qq22,rinv22);
561 felec = _mm_mul_ps(velec,rinvsq22);
563 /* Update potential sum for this i atom from the interaction with this j atom. */
564 velecsum = _mm_add_ps(velecsum,velec);
568 /* Calculate temporary vectorial force */
569 tx = _mm_mul_ps(fscal,dx22);
570 ty = _mm_mul_ps(fscal,dy22);
571 tz = _mm_mul_ps(fscal,dz22);
573 /* Update vectorial force */
574 fix2 = _mm_add_ps(fix2,tx);
575 fiy2 = _mm_add_ps(fiy2,ty);
576 fiz2 = _mm_add_ps(fiz2,tz);
578 fjx2 = _mm_add_ps(fjx2,tx);
579 fjy2 = _mm_add_ps(fjy2,ty);
580 fjz2 = _mm_add_ps(fjz2,tz);
582 fjptrA = f+j_coord_offsetA;
583 fjptrB = f+j_coord_offsetB;
584 fjptrC = f+j_coord_offsetC;
585 fjptrD = f+j_coord_offsetD;
587 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
588 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
590 /* Inner loop uses 287 flops */
596 /* Get j neighbor index, and coordinate index */
597 jnrlistA = jjnr[jidx];
598 jnrlistB = jjnr[jidx+1];
599 jnrlistC = jjnr[jidx+2];
600 jnrlistD = jjnr[jidx+3];
601 /* Sign of each element will be negative for non-real atoms.
602 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
603 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
605 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
606 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
607 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
608 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
609 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
610 j_coord_offsetA = DIM*jnrA;
611 j_coord_offsetB = DIM*jnrB;
612 j_coord_offsetC = DIM*jnrC;
613 j_coord_offsetD = DIM*jnrD;
615 /* load j atom coordinates */
616 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
617 x+j_coord_offsetC,x+j_coord_offsetD,
618 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
620 /* Calculate displacement vector */
621 dx00 = _mm_sub_ps(ix0,jx0);
622 dy00 = _mm_sub_ps(iy0,jy0);
623 dz00 = _mm_sub_ps(iz0,jz0);
624 dx01 = _mm_sub_ps(ix0,jx1);
625 dy01 = _mm_sub_ps(iy0,jy1);
626 dz01 = _mm_sub_ps(iz0,jz1);
627 dx02 = _mm_sub_ps(ix0,jx2);
628 dy02 = _mm_sub_ps(iy0,jy2);
629 dz02 = _mm_sub_ps(iz0,jz2);
630 dx10 = _mm_sub_ps(ix1,jx0);
631 dy10 = _mm_sub_ps(iy1,jy0);
632 dz10 = _mm_sub_ps(iz1,jz0);
633 dx11 = _mm_sub_ps(ix1,jx1);
634 dy11 = _mm_sub_ps(iy1,jy1);
635 dz11 = _mm_sub_ps(iz1,jz1);
636 dx12 = _mm_sub_ps(ix1,jx2);
637 dy12 = _mm_sub_ps(iy1,jy2);
638 dz12 = _mm_sub_ps(iz1,jz2);
639 dx20 = _mm_sub_ps(ix2,jx0);
640 dy20 = _mm_sub_ps(iy2,jy0);
641 dz20 = _mm_sub_ps(iz2,jz0);
642 dx21 = _mm_sub_ps(ix2,jx1);
643 dy21 = _mm_sub_ps(iy2,jy1);
644 dz21 = _mm_sub_ps(iz2,jz1);
645 dx22 = _mm_sub_ps(ix2,jx2);
646 dy22 = _mm_sub_ps(iy2,jy2);
647 dz22 = _mm_sub_ps(iz2,jz2);
649 /* Calculate squared distance and things based on it */
650 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
651 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
652 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
653 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
654 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
655 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
656 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
657 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
658 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
660 rinv00 = gmx_mm_invsqrt_ps(rsq00);
661 rinv01 = gmx_mm_invsqrt_ps(rsq01);
662 rinv02 = gmx_mm_invsqrt_ps(rsq02);
663 rinv10 = gmx_mm_invsqrt_ps(rsq10);
664 rinv11 = gmx_mm_invsqrt_ps(rsq11);
665 rinv12 = gmx_mm_invsqrt_ps(rsq12);
666 rinv20 = gmx_mm_invsqrt_ps(rsq20);
667 rinv21 = gmx_mm_invsqrt_ps(rsq21);
668 rinv22 = gmx_mm_invsqrt_ps(rsq22);
670 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
671 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
672 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
673 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
674 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
675 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
676 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
677 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
678 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
680 fjx0 = _mm_setzero_ps();
681 fjy0 = _mm_setzero_ps();
682 fjz0 = _mm_setzero_ps();
683 fjx1 = _mm_setzero_ps();
684 fjy1 = _mm_setzero_ps();
685 fjz1 = _mm_setzero_ps();
686 fjx2 = _mm_setzero_ps();
687 fjy2 = _mm_setzero_ps();
688 fjz2 = _mm_setzero_ps();
690 /**************************
691 * CALCULATE INTERACTIONS *
692 **************************/
694 r00 = _mm_mul_ps(rsq00,rinv00);
695 r00 = _mm_andnot_ps(dummy_mask,r00);
697 /* Calculate table index by multiplying r with table scale and truncate to integer */
698 rt = _mm_mul_ps(r00,vftabscale);
699 vfitab = _mm_cvttps_epi32(rt);
700 vfeps = _mm_sub_ps(rt,_mm_cvtepi32_ps(vfitab));
701 vfitab = _mm_slli_epi32(vfitab,3);
703 /* COULOMB ELECTROSTATICS */
704 velec = _mm_mul_ps(qq00,rinv00);
705 felec = _mm_mul_ps(velec,rinvsq00);
707 /* CUBIC SPLINE TABLE DISPERSION */
708 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
709 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
710 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
711 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
712 _MM_TRANSPOSE4_PS(Y,F,G,H);
713 Heps = _mm_mul_ps(vfeps,H);
714 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
715 VV = _mm_add_ps(Y,_mm_mul_ps(vfeps,Fp));
716 vvdw6 = _mm_mul_ps(c6_00,VV);
717 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
718 fvdw6 = _mm_mul_ps(c6_00,FF);
720 /* CUBIC SPLINE TABLE REPULSION */
721 vfitab = _mm_add_epi32(vfitab,ifour);
722 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
723 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
724 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
725 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
726 _MM_TRANSPOSE4_PS(Y,F,G,H);
727 Heps = _mm_mul_ps(vfeps,H);
728 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
729 VV = _mm_add_ps(Y,_mm_mul_ps(vfeps,Fp));
730 vvdw12 = _mm_mul_ps(c12_00,VV);
731 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
732 fvdw12 = _mm_mul_ps(c12_00,FF);
733 vvdw = _mm_add_ps(vvdw12,vvdw6);
734 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
736 /* Update potential sum for this i atom from the interaction with this j atom. */
737 velec = _mm_andnot_ps(dummy_mask,velec);
738 velecsum = _mm_add_ps(velecsum,velec);
739 vvdw = _mm_andnot_ps(dummy_mask,vvdw);
740 vvdwsum = _mm_add_ps(vvdwsum,vvdw);
742 fscal = _mm_add_ps(felec,fvdw);
744 fscal = _mm_andnot_ps(dummy_mask,fscal);
746 /* Calculate temporary vectorial force */
747 tx = _mm_mul_ps(fscal,dx00);
748 ty = _mm_mul_ps(fscal,dy00);
749 tz = _mm_mul_ps(fscal,dz00);
751 /* Update vectorial force */
752 fix0 = _mm_add_ps(fix0,tx);
753 fiy0 = _mm_add_ps(fiy0,ty);
754 fiz0 = _mm_add_ps(fiz0,tz);
756 fjx0 = _mm_add_ps(fjx0,tx);
757 fjy0 = _mm_add_ps(fjy0,ty);
758 fjz0 = _mm_add_ps(fjz0,tz);
760 /**************************
761 * CALCULATE INTERACTIONS *
762 **************************/
764 /* COULOMB ELECTROSTATICS */
765 velec = _mm_mul_ps(qq01,rinv01);
766 felec = _mm_mul_ps(velec,rinvsq01);
768 /* Update potential sum for this i atom from the interaction with this j atom. */
769 velec = _mm_andnot_ps(dummy_mask,velec);
770 velecsum = _mm_add_ps(velecsum,velec);
774 fscal = _mm_andnot_ps(dummy_mask,fscal);
776 /* Calculate temporary vectorial force */
777 tx = _mm_mul_ps(fscal,dx01);
778 ty = _mm_mul_ps(fscal,dy01);
779 tz = _mm_mul_ps(fscal,dz01);
781 /* Update vectorial force */
782 fix0 = _mm_add_ps(fix0,tx);
783 fiy0 = _mm_add_ps(fiy0,ty);
784 fiz0 = _mm_add_ps(fiz0,tz);
786 fjx1 = _mm_add_ps(fjx1,tx);
787 fjy1 = _mm_add_ps(fjy1,ty);
788 fjz1 = _mm_add_ps(fjz1,tz);
790 /**************************
791 * CALCULATE INTERACTIONS *
792 **************************/
794 /* COULOMB ELECTROSTATICS */
795 velec = _mm_mul_ps(qq02,rinv02);
796 felec = _mm_mul_ps(velec,rinvsq02);
798 /* Update potential sum for this i atom from the interaction with this j atom. */
799 velec = _mm_andnot_ps(dummy_mask,velec);
800 velecsum = _mm_add_ps(velecsum,velec);
804 fscal = _mm_andnot_ps(dummy_mask,fscal);
806 /* Calculate temporary vectorial force */
807 tx = _mm_mul_ps(fscal,dx02);
808 ty = _mm_mul_ps(fscal,dy02);
809 tz = _mm_mul_ps(fscal,dz02);
811 /* Update vectorial force */
812 fix0 = _mm_add_ps(fix0,tx);
813 fiy0 = _mm_add_ps(fiy0,ty);
814 fiz0 = _mm_add_ps(fiz0,tz);
816 fjx2 = _mm_add_ps(fjx2,tx);
817 fjy2 = _mm_add_ps(fjy2,ty);
818 fjz2 = _mm_add_ps(fjz2,tz);
820 /**************************
821 * CALCULATE INTERACTIONS *
822 **************************/
824 /* COULOMB ELECTROSTATICS */
825 velec = _mm_mul_ps(qq10,rinv10);
826 felec = _mm_mul_ps(velec,rinvsq10);
828 /* Update potential sum for this i atom from the interaction with this j atom. */
829 velec = _mm_andnot_ps(dummy_mask,velec);
830 velecsum = _mm_add_ps(velecsum,velec);
834 fscal = _mm_andnot_ps(dummy_mask,fscal);
836 /* Calculate temporary vectorial force */
837 tx = _mm_mul_ps(fscal,dx10);
838 ty = _mm_mul_ps(fscal,dy10);
839 tz = _mm_mul_ps(fscal,dz10);
841 /* Update vectorial force */
842 fix1 = _mm_add_ps(fix1,tx);
843 fiy1 = _mm_add_ps(fiy1,ty);
844 fiz1 = _mm_add_ps(fiz1,tz);
846 fjx0 = _mm_add_ps(fjx0,tx);
847 fjy0 = _mm_add_ps(fjy0,ty);
848 fjz0 = _mm_add_ps(fjz0,tz);
850 /**************************
851 * CALCULATE INTERACTIONS *
852 **************************/
854 /* COULOMB ELECTROSTATICS */
855 velec = _mm_mul_ps(qq11,rinv11);
856 felec = _mm_mul_ps(velec,rinvsq11);
858 /* Update potential sum for this i atom from the interaction with this j atom. */
859 velec = _mm_andnot_ps(dummy_mask,velec);
860 velecsum = _mm_add_ps(velecsum,velec);
864 fscal = _mm_andnot_ps(dummy_mask,fscal);
866 /* Calculate temporary vectorial force */
867 tx = _mm_mul_ps(fscal,dx11);
868 ty = _mm_mul_ps(fscal,dy11);
869 tz = _mm_mul_ps(fscal,dz11);
871 /* Update vectorial force */
872 fix1 = _mm_add_ps(fix1,tx);
873 fiy1 = _mm_add_ps(fiy1,ty);
874 fiz1 = _mm_add_ps(fiz1,tz);
876 fjx1 = _mm_add_ps(fjx1,tx);
877 fjy1 = _mm_add_ps(fjy1,ty);
878 fjz1 = _mm_add_ps(fjz1,tz);
880 /**************************
881 * CALCULATE INTERACTIONS *
882 **************************/
884 /* COULOMB ELECTROSTATICS */
885 velec = _mm_mul_ps(qq12,rinv12);
886 felec = _mm_mul_ps(velec,rinvsq12);
888 /* Update potential sum for this i atom from the interaction with this j atom. */
889 velec = _mm_andnot_ps(dummy_mask,velec);
890 velecsum = _mm_add_ps(velecsum,velec);
894 fscal = _mm_andnot_ps(dummy_mask,fscal);
896 /* Calculate temporary vectorial force */
897 tx = _mm_mul_ps(fscal,dx12);
898 ty = _mm_mul_ps(fscal,dy12);
899 tz = _mm_mul_ps(fscal,dz12);
901 /* Update vectorial force */
902 fix1 = _mm_add_ps(fix1,tx);
903 fiy1 = _mm_add_ps(fiy1,ty);
904 fiz1 = _mm_add_ps(fiz1,tz);
906 fjx2 = _mm_add_ps(fjx2,tx);
907 fjy2 = _mm_add_ps(fjy2,ty);
908 fjz2 = _mm_add_ps(fjz2,tz);
910 /**************************
911 * CALCULATE INTERACTIONS *
912 **************************/
914 /* COULOMB ELECTROSTATICS */
915 velec = _mm_mul_ps(qq20,rinv20);
916 felec = _mm_mul_ps(velec,rinvsq20);
918 /* Update potential sum for this i atom from the interaction with this j atom. */
919 velec = _mm_andnot_ps(dummy_mask,velec);
920 velecsum = _mm_add_ps(velecsum,velec);
924 fscal = _mm_andnot_ps(dummy_mask,fscal);
926 /* Calculate temporary vectorial force */
927 tx = _mm_mul_ps(fscal,dx20);
928 ty = _mm_mul_ps(fscal,dy20);
929 tz = _mm_mul_ps(fscal,dz20);
931 /* Update vectorial force */
932 fix2 = _mm_add_ps(fix2,tx);
933 fiy2 = _mm_add_ps(fiy2,ty);
934 fiz2 = _mm_add_ps(fiz2,tz);
936 fjx0 = _mm_add_ps(fjx0,tx);
937 fjy0 = _mm_add_ps(fjy0,ty);
938 fjz0 = _mm_add_ps(fjz0,tz);
940 /**************************
941 * CALCULATE INTERACTIONS *
942 **************************/
944 /* COULOMB ELECTROSTATICS */
945 velec = _mm_mul_ps(qq21,rinv21);
946 felec = _mm_mul_ps(velec,rinvsq21);
948 /* Update potential sum for this i atom from the interaction with this j atom. */
949 velec = _mm_andnot_ps(dummy_mask,velec);
950 velecsum = _mm_add_ps(velecsum,velec);
954 fscal = _mm_andnot_ps(dummy_mask,fscal);
956 /* Calculate temporary vectorial force */
957 tx = _mm_mul_ps(fscal,dx21);
958 ty = _mm_mul_ps(fscal,dy21);
959 tz = _mm_mul_ps(fscal,dz21);
961 /* Update vectorial force */
962 fix2 = _mm_add_ps(fix2,tx);
963 fiy2 = _mm_add_ps(fiy2,ty);
964 fiz2 = _mm_add_ps(fiz2,tz);
966 fjx1 = _mm_add_ps(fjx1,tx);
967 fjy1 = _mm_add_ps(fjy1,ty);
968 fjz1 = _mm_add_ps(fjz1,tz);
970 /**************************
971 * CALCULATE INTERACTIONS *
972 **************************/
974 /* COULOMB ELECTROSTATICS */
975 velec = _mm_mul_ps(qq22,rinv22);
976 felec = _mm_mul_ps(velec,rinvsq22);
978 /* Update potential sum for this i atom from the interaction with this j atom. */
979 velec = _mm_andnot_ps(dummy_mask,velec);
980 velecsum = _mm_add_ps(velecsum,velec);
984 fscal = _mm_andnot_ps(dummy_mask,fscal);
986 /* Calculate temporary vectorial force */
987 tx = _mm_mul_ps(fscal,dx22);
988 ty = _mm_mul_ps(fscal,dy22);
989 tz = _mm_mul_ps(fscal,dz22);
991 /* Update vectorial force */
992 fix2 = _mm_add_ps(fix2,tx);
993 fiy2 = _mm_add_ps(fiy2,ty);
994 fiz2 = _mm_add_ps(fiz2,tz);
996 fjx2 = _mm_add_ps(fjx2,tx);
997 fjy2 = _mm_add_ps(fjy2,ty);
998 fjz2 = _mm_add_ps(fjz2,tz);
1000 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1001 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1002 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1003 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1005 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1006 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1008 /* Inner loop uses 288 flops */
1011 /* End of innermost loop */
1013 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1014 f+i_coord_offset,fshift+i_shift_offset);
1017 /* Update potential energies */
1018 gmx_mm_update_1pot_ps(velecsum,kernel_data->energygrp_elec+ggid);
1019 gmx_mm_update_1pot_ps(vvdwsum,kernel_data->energygrp_vdw+ggid);
1021 /* Increment number of inner iterations */
1022 inneriter += j_index_end - j_index_start;
1024 /* Outer loop uses 20 flops */
1027 /* Increment number of outer iterations */
1030 /* Update outer/inner flops */
1032 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_VF,outeriter*20 + inneriter*288);
1035 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_F_sse2_single
1036 * Electrostatics interaction: Coulomb
1037 * VdW interaction: CubicSplineTable
1038 * Geometry: Water3-Water3
1039 * Calculate force/pot: Force
1042 nb_kernel_ElecCoul_VdwCSTab_GeomW3W3_F_sse2_single
1043 (t_nblist * gmx_restrict nlist,
1044 rvec * gmx_restrict xx,
1045 rvec * gmx_restrict ff,
1046 t_forcerec * gmx_restrict fr,
1047 t_mdatoms * gmx_restrict mdatoms,
1048 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1049 t_nrnb * gmx_restrict nrnb)
1051 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1052 * just 0 for non-waters.
1053 * Suffixes A,B,C,D refer to j loop unrolling done with SSE, e.g. for the four different
1054 * jnr indices corresponding to data put in the four positions in the SIMD register.
1056 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1057 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1058 int jnrA,jnrB,jnrC,jnrD;
1059 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1060 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1061 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1062 real rcutoff_scalar;
1063 real *shiftvec,*fshift,*x,*f;
1064 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1065 real scratch[4*DIM];
1066 __m128 tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
1068 __m128 ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1070 __m128 ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1072 __m128 ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1073 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1074 __m128 jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1075 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1076 __m128 jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1077 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1078 __m128 jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1079 __m128 dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1080 __m128 dx01,dy01,dz01,rsq01,rinv01,rinvsq01,r01,qq01,c6_01,c12_01;
1081 __m128 dx02,dy02,dz02,rsq02,rinv02,rinvsq02,r02,qq02,c6_02,c12_02;
1082 __m128 dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
1083 __m128 dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1084 __m128 dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1085 __m128 dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
1086 __m128 dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1087 __m128 dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1088 __m128 velec,felec,velecsum,facel,crf,krf,krf2;
1091 __m128 rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1094 __m128 one_sixth = _mm_set1_ps(1.0/6.0);
1095 __m128 one_twelfth = _mm_set1_ps(1.0/12.0);
1097 __m128i ifour = _mm_set1_epi32(4);
1098 __m128 rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
1100 __m128 dummy_mask,cutoff_mask;
1101 __m128 signbit = _mm_castsi128_ps( _mm_set1_epi32(0x80000000) );
1102 __m128 one = _mm_set1_ps(1.0);
1103 __m128 two = _mm_set1_ps(2.0);
1109 jindex = nlist->jindex;
1111 shiftidx = nlist->shift;
1113 shiftvec = fr->shift_vec[0];
1114 fshift = fr->fshift[0];
1115 facel = _mm_set1_ps(fr->epsfac);
1116 charge = mdatoms->chargeA;
1117 nvdwtype = fr->ntype;
1118 vdwparam = fr->nbfp;
1119 vdwtype = mdatoms->typeA;
1121 vftab = kernel_data->table_vdw->data;
1122 vftabscale = _mm_set1_ps(kernel_data->table_vdw->scale);
1124 /* Setup water-specific parameters */
1125 inr = nlist->iinr[0];
1126 iq0 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+0]));
1127 iq1 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+1]));
1128 iq2 = _mm_mul_ps(facel,_mm_set1_ps(charge[inr+2]));
1129 vdwioffset0 = 2*nvdwtype*vdwtype[inr+0];
1131 jq0 = _mm_set1_ps(charge[inr+0]);
1132 jq1 = _mm_set1_ps(charge[inr+1]);
1133 jq2 = _mm_set1_ps(charge[inr+2]);
1134 vdwjidx0A = 2*vdwtype[inr+0];
1135 qq00 = _mm_mul_ps(iq0,jq0);
1136 c6_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A]);
1137 c12_00 = _mm_set1_ps(vdwparam[vdwioffset0+vdwjidx0A+1]);
1138 qq01 = _mm_mul_ps(iq0,jq1);
1139 qq02 = _mm_mul_ps(iq0,jq2);
1140 qq10 = _mm_mul_ps(iq1,jq0);
1141 qq11 = _mm_mul_ps(iq1,jq1);
1142 qq12 = _mm_mul_ps(iq1,jq2);
1143 qq20 = _mm_mul_ps(iq2,jq0);
1144 qq21 = _mm_mul_ps(iq2,jq1);
1145 qq22 = _mm_mul_ps(iq2,jq2);
1147 /* Avoid stupid compiler warnings */
1148 jnrA = jnrB = jnrC = jnrD = 0;
1149 j_coord_offsetA = 0;
1150 j_coord_offsetB = 0;
1151 j_coord_offsetC = 0;
1152 j_coord_offsetD = 0;
1157 for(iidx=0;iidx<4*DIM;iidx++)
1159 scratch[iidx] = 0.0;
1162 /* Start outer loop over neighborlists */
1163 for(iidx=0; iidx<nri; iidx++)
1165 /* Load shift vector for this list */
1166 i_shift_offset = DIM*shiftidx[iidx];
1168 /* Load limits for loop over neighbors */
1169 j_index_start = jindex[iidx];
1170 j_index_end = jindex[iidx+1];
1172 /* Get outer coordinate index */
1174 i_coord_offset = DIM*inr;
1176 /* Load i particle coords and add shift vector */
1177 gmx_mm_load_shift_and_3rvec_broadcast_ps(shiftvec+i_shift_offset,x+i_coord_offset,
1178 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2);
1180 fix0 = _mm_setzero_ps();
1181 fiy0 = _mm_setzero_ps();
1182 fiz0 = _mm_setzero_ps();
1183 fix1 = _mm_setzero_ps();
1184 fiy1 = _mm_setzero_ps();
1185 fiz1 = _mm_setzero_ps();
1186 fix2 = _mm_setzero_ps();
1187 fiy2 = _mm_setzero_ps();
1188 fiz2 = _mm_setzero_ps();
1190 /* Start inner kernel loop */
1191 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1194 /* Get j neighbor index, and coordinate index */
1196 jnrB = jjnr[jidx+1];
1197 jnrC = jjnr[jidx+2];
1198 jnrD = jjnr[jidx+3];
1199 j_coord_offsetA = DIM*jnrA;
1200 j_coord_offsetB = DIM*jnrB;
1201 j_coord_offsetC = DIM*jnrC;
1202 j_coord_offsetD = DIM*jnrD;
1204 /* load j atom coordinates */
1205 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1206 x+j_coord_offsetC,x+j_coord_offsetD,
1207 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1209 /* Calculate displacement vector */
1210 dx00 = _mm_sub_ps(ix0,jx0);
1211 dy00 = _mm_sub_ps(iy0,jy0);
1212 dz00 = _mm_sub_ps(iz0,jz0);
1213 dx01 = _mm_sub_ps(ix0,jx1);
1214 dy01 = _mm_sub_ps(iy0,jy1);
1215 dz01 = _mm_sub_ps(iz0,jz1);
1216 dx02 = _mm_sub_ps(ix0,jx2);
1217 dy02 = _mm_sub_ps(iy0,jy2);
1218 dz02 = _mm_sub_ps(iz0,jz2);
1219 dx10 = _mm_sub_ps(ix1,jx0);
1220 dy10 = _mm_sub_ps(iy1,jy0);
1221 dz10 = _mm_sub_ps(iz1,jz0);
1222 dx11 = _mm_sub_ps(ix1,jx1);
1223 dy11 = _mm_sub_ps(iy1,jy1);
1224 dz11 = _mm_sub_ps(iz1,jz1);
1225 dx12 = _mm_sub_ps(ix1,jx2);
1226 dy12 = _mm_sub_ps(iy1,jy2);
1227 dz12 = _mm_sub_ps(iz1,jz2);
1228 dx20 = _mm_sub_ps(ix2,jx0);
1229 dy20 = _mm_sub_ps(iy2,jy0);
1230 dz20 = _mm_sub_ps(iz2,jz0);
1231 dx21 = _mm_sub_ps(ix2,jx1);
1232 dy21 = _mm_sub_ps(iy2,jy1);
1233 dz21 = _mm_sub_ps(iz2,jz1);
1234 dx22 = _mm_sub_ps(ix2,jx2);
1235 dy22 = _mm_sub_ps(iy2,jy2);
1236 dz22 = _mm_sub_ps(iz2,jz2);
1238 /* Calculate squared distance and things based on it */
1239 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1240 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1241 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1242 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1243 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1244 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1245 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1246 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1247 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1249 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1250 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1251 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1252 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1253 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1254 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1255 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1256 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1257 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1259 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1260 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1261 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1262 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1263 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1264 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1265 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1266 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1267 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1269 fjx0 = _mm_setzero_ps();
1270 fjy0 = _mm_setzero_ps();
1271 fjz0 = _mm_setzero_ps();
1272 fjx1 = _mm_setzero_ps();
1273 fjy1 = _mm_setzero_ps();
1274 fjz1 = _mm_setzero_ps();
1275 fjx2 = _mm_setzero_ps();
1276 fjy2 = _mm_setzero_ps();
1277 fjz2 = _mm_setzero_ps();
1279 /**************************
1280 * CALCULATE INTERACTIONS *
1281 **************************/
1283 r00 = _mm_mul_ps(rsq00,rinv00);
1285 /* Calculate table index by multiplying r with table scale and truncate to integer */
1286 rt = _mm_mul_ps(r00,vftabscale);
1287 vfitab = _mm_cvttps_epi32(rt);
1288 vfeps = _mm_sub_ps(rt,_mm_cvtepi32_ps(vfitab));
1289 vfitab = _mm_slli_epi32(vfitab,3);
1291 /* COULOMB ELECTROSTATICS */
1292 velec = _mm_mul_ps(qq00,rinv00);
1293 felec = _mm_mul_ps(velec,rinvsq00);
1295 /* CUBIC SPLINE TABLE DISPERSION */
1296 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
1297 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
1298 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
1299 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
1300 _MM_TRANSPOSE4_PS(Y,F,G,H);
1301 Heps = _mm_mul_ps(vfeps,H);
1302 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
1303 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
1304 fvdw6 = _mm_mul_ps(c6_00,FF);
1306 /* CUBIC SPLINE TABLE REPULSION */
1307 vfitab = _mm_add_epi32(vfitab,ifour);
1308 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
1309 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
1310 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
1311 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
1312 _MM_TRANSPOSE4_PS(Y,F,G,H);
1313 Heps = _mm_mul_ps(vfeps,H);
1314 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
1315 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
1316 fvdw12 = _mm_mul_ps(c12_00,FF);
1317 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
1319 fscal = _mm_add_ps(felec,fvdw);
1321 /* Calculate temporary vectorial force */
1322 tx = _mm_mul_ps(fscal,dx00);
1323 ty = _mm_mul_ps(fscal,dy00);
1324 tz = _mm_mul_ps(fscal,dz00);
1326 /* Update vectorial force */
1327 fix0 = _mm_add_ps(fix0,tx);
1328 fiy0 = _mm_add_ps(fiy0,ty);
1329 fiz0 = _mm_add_ps(fiz0,tz);
1331 fjx0 = _mm_add_ps(fjx0,tx);
1332 fjy0 = _mm_add_ps(fjy0,ty);
1333 fjz0 = _mm_add_ps(fjz0,tz);
1335 /**************************
1336 * CALCULATE INTERACTIONS *
1337 **************************/
1339 /* COULOMB ELECTROSTATICS */
1340 velec = _mm_mul_ps(qq01,rinv01);
1341 felec = _mm_mul_ps(velec,rinvsq01);
1345 /* Calculate temporary vectorial force */
1346 tx = _mm_mul_ps(fscal,dx01);
1347 ty = _mm_mul_ps(fscal,dy01);
1348 tz = _mm_mul_ps(fscal,dz01);
1350 /* Update vectorial force */
1351 fix0 = _mm_add_ps(fix0,tx);
1352 fiy0 = _mm_add_ps(fiy0,ty);
1353 fiz0 = _mm_add_ps(fiz0,tz);
1355 fjx1 = _mm_add_ps(fjx1,tx);
1356 fjy1 = _mm_add_ps(fjy1,ty);
1357 fjz1 = _mm_add_ps(fjz1,tz);
1359 /**************************
1360 * CALCULATE INTERACTIONS *
1361 **************************/
1363 /* COULOMB ELECTROSTATICS */
1364 velec = _mm_mul_ps(qq02,rinv02);
1365 felec = _mm_mul_ps(velec,rinvsq02);
1369 /* Calculate temporary vectorial force */
1370 tx = _mm_mul_ps(fscal,dx02);
1371 ty = _mm_mul_ps(fscal,dy02);
1372 tz = _mm_mul_ps(fscal,dz02);
1374 /* Update vectorial force */
1375 fix0 = _mm_add_ps(fix0,tx);
1376 fiy0 = _mm_add_ps(fiy0,ty);
1377 fiz0 = _mm_add_ps(fiz0,tz);
1379 fjx2 = _mm_add_ps(fjx2,tx);
1380 fjy2 = _mm_add_ps(fjy2,ty);
1381 fjz2 = _mm_add_ps(fjz2,tz);
1383 /**************************
1384 * CALCULATE INTERACTIONS *
1385 **************************/
1387 /* COULOMB ELECTROSTATICS */
1388 velec = _mm_mul_ps(qq10,rinv10);
1389 felec = _mm_mul_ps(velec,rinvsq10);
1393 /* Calculate temporary vectorial force */
1394 tx = _mm_mul_ps(fscal,dx10);
1395 ty = _mm_mul_ps(fscal,dy10);
1396 tz = _mm_mul_ps(fscal,dz10);
1398 /* Update vectorial force */
1399 fix1 = _mm_add_ps(fix1,tx);
1400 fiy1 = _mm_add_ps(fiy1,ty);
1401 fiz1 = _mm_add_ps(fiz1,tz);
1403 fjx0 = _mm_add_ps(fjx0,tx);
1404 fjy0 = _mm_add_ps(fjy0,ty);
1405 fjz0 = _mm_add_ps(fjz0,tz);
1407 /**************************
1408 * CALCULATE INTERACTIONS *
1409 **************************/
1411 /* COULOMB ELECTROSTATICS */
1412 velec = _mm_mul_ps(qq11,rinv11);
1413 felec = _mm_mul_ps(velec,rinvsq11);
1417 /* Calculate temporary vectorial force */
1418 tx = _mm_mul_ps(fscal,dx11);
1419 ty = _mm_mul_ps(fscal,dy11);
1420 tz = _mm_mul_ps(fscal,dz11);
1422 /* Update vectorial force */
1423 fix1 = _mm_add_ps(fix1,tx);
1424 fiy1 = _mm_add_ps(fiy1,ty);
1425 fiz1 = _mm_add_ps(fiz1,tz);
1427 fjx1 = _mm_add_ps(fjx1,tx);
1428 fjy1 = _mm_add_ps(fjy1,ty);
1429 fjz1 = _mm_add_ps(fjz1,tz);
1431 /**************************
1432 * CALCULATE INTERACTIONS *
1433 **************************/
1435 /* COULOMB ELECTROSTATICS */
1436 velec = _mm_mul_ps(qq12,rinv12);
1437 felec = _mm_mul_ps(velec,rinvsq12);
1441 /* Calculate temporary vectorial force */
1442 tx = _mm_mul_ps(fscal,dx12);
1443 ty = _mm_mul_ps(fscal,dy12);
1444 tz = _mm_mul_ps(fscal,dz12);
1446 /* Update vectorial force */
1447 fix1 = _mm_add_ps(fix1,tx);
1448 fiy1 = _mm_add_ps(fiy1,ty);
1449 fiz1 = _mm_add_ps(fiz1,tz);
1451 fjx2 = _mm_add_ps(fjx2,tx);
1452 fjy2 = _mm_add_ps(fjy2,ty);
1453 fjz2 = _mm_add_ps(fjz2,tz);
1455 /**************************
1456 * CALCULATE INTERACTIONS *
1457 **************************/
1459 /* COULOMB ELECTROSTATICS */
1460 velec = _mm_mul_ps(qq20,rinv20);
1461 felec = _mm_mul_ps(velec,rinvsq20);
1465 /* Calculate temporary vectorial force */
1466 tx = _mm_mul_ps(fscal,dx20);
1467 ty = _mm_mul_ps(fscal,dy20);
1468 tz = _mm_mul_ps(fscal,dz20);
1470 /* Update vectorial force */
1471 fix2 = _mm_add_ps(fix2,tx);
1472 fiy2 = _mm_add_ps(fiy2,ty);
1473 fiz2 = _mm_add_ps(fiz2,tz);
1475 fjx0 = _mm_add_ps(fjx0,tx);
1476 fjy0 = _mm_add_ps(fjy0,ty);
1477 fjz0 = _mm_add_ps(fjz0,tz);
1479 /**************************
1480 * CALCULATE INTERACTIONS *
1481 **************************/
1483 /* COULOMB ELECTROSTATICS */
1484 velec = _mm_mul_ps(qq21,rinv21);
1485 felec = _mm_mul_ps(velec,rinvsq21);
1489 /* Calculate temporary vectorial force */
1490 tx = _mm_mul_ps(fscal,dx21);
1491 ty = _mm_mul_ps(fscal,dy21);
1492 tz = _mm_mul_ps(fscal,dz21);
1494 /* Update vectorial force */
1495 fix2 = _mm_add_ps(fix2,tx);
1496 fiy2 = _mm_add_ps(fiy2,ty);
1497 fiz2 = _mm_add_ps(fiz2,tz);
1499 fjx1 = _mm_add_ps(fjx1,tx);
1500 fjy1 = _mm_add_ps(fjy1,ty);
1501 fjz1 = _mm_add_ps(fjz1,tz);
1503 /**************************
1504 * CALCULATE INTERACTIONS *
1505 **************************/
1507 /* COULOMB ELECTROSTATICS */
1508 velec = _mm_mul_ps(qq22,rinv22);
1509 felec = _mm_mul_ps(velec,rinvsq22);
1513 /* Calculate temporary vectorial force */
1514 tx = _mm_mul_ps(fscal,dx22);
1515 ty = _mm_mul_ps(fscal,dy22);
1516 tz = _mm_mul_ps(fscal,dz22);
1518 /* Update vectorial force */
1519 fix2 = _mm_add_ps(fix2,tx);
1520 fiy2 = _mm_add_ps(fiy2,ty);
1521 fiz2 = _mm_add_ps(fiz2,tz);
1523 fjx2 = _mm_add_ps(fjx2,tx);
1524 fjy2 = _mm_add_ps(fjy2,ty);
1525 fjz2 = _mm_add_ps(fjz2,tz);
1527 fjptrA = f+j_coord_offsetA;
1528 fjptrB = f+j_coord_offsetB;
1529 fjptrC = f+j_coord_offsetC;
1530 fjptrD = f+j_coord_offsetD;
1532 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1533 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1535 /* Inner loop uses 270 flops */
1538 if(jidx<j_index_end)
1541 /* Get j neighbor index, and coordinate index */
1542 jnrlistA = jjnr[jidx];
1543 jnrlistB = jjnr[jidx+1];
1544 jnrlistC = jjnr[jidx+2];
1545 jnrlistD = jjnr[jidx+3];
1546 /* Sign of each element will be negative for non-real atoms.
1547 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1548 * so use it as val = _mm_andnot_ps(mask,val) to clear dummy entries.
1550 dummy_mask = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1551 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1552 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1553 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1554 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1555 j_coord_offsetA = DIM*jnrA;
1556 j_coord_offsetB = DIM*jnrB;
1557 j_coord_offsetC = DIM*jnrC;
1558 j_coord_offsetD = DIM*jnrD;
1560 /* load j atom coordinates */
1561 gmx_mm_load_3rvec_4ptr_swizzle_ps(x+j_coord_offsetA,x+j_coord_offsetB,
1562 x+j_coord_offsetC,x+j_coord_offsetD,
1563 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,&jy2,&jz2);
1565 /* Calculate displacement vector */
1566 dx00 = _mm_sub_ps(ix0,jx0);
1567 dy00 = _mm_sub_ps(iy0,jy0);
1568 dz00 = _mm_sub_ps(iz0,jz0);
1569 dx01 = _mm_sub_ps(ix0,jx1);
1570 dy01 = _mm_sub_ps(iy0,jy1);
1571 dz01 = _mm_sub_ps(iz0,jz1);
1572 dx02 = _mm_sub_ps(ix0,jx2);
1573 dy02 = _mm_sub_ps(iy0,jy2);
1574 dz02 = _mm_sub_ps(iz0,jz2);
1575 dx10 = _mm_sub_ps(ix1,jx0);
1576 dy10 = _mm_sub_ps(iy1,jy0);
1577 dz10 = _mm_sub_ps(iz1,jz0);
1578 dx11 = _mm_sub_ps(ix1,jx1);
1579 dy11 = _mm_sub_ps(iy1,jy1);
1580 dz11 = _mm_sub_ps(iz1,jz1);
1581 dx12 = _mm_sub_ps(ix1,jx2);
1582 dy12 = _mm_sub_ps(iy1,jy2);
1583 dz12 = _mm_sub_ps(iz1,jz2);
1584 dx20 = _mm_sub_ps(ix2,jx0);
1585 dy20 = _mm_sub_ps(iy2,jy0);
1586 dz20 = _mm_sub_ps(iz2,jz0);
1587 dx21 = _mm_sub_ps(ix2,jx1);
1588 dy21 = _mm_sub_ps(iy2,jy1);
1589 dz21 = _mm_sub_ps(iz2,jz1);
1590 dx22 = _mm_sub_ps(ix2,jx2);
1591 dy22 = _mm_sub_ps(iy2,jy2);
1592 dz22 = _mm_sub_ps(iz2,jz2);
1594 /* Calculate squared distance and things based on it */
1595 rsq00 = gmx_mm_calc_rsq_ps(dx00,dy00,dz00);
1596 rsq01 = gmx_mm_calc_rsq_ps(dx01,dy01,dz01);
1597 rsq02 = gmx_mm_calc_rsq_ps(dx02,dy02,dz02);
1598 rsq10 = gmx_mm_calc_rsq_ps(dx10,dy10,dz10);
1599 rsq11 = gmx_mm_calc_rsq_ps(dx11,dy11,dz11);
1600 rsq12 = gmx_mm_calc_rsq_ps(dx12,dy12,dz12);
1601 rsq20 = gmx_mm_calc_rsq_ps(dx20,dy20,dz20);
1602 rsq21 = gmx_mm_calc_rsq_ps(dx21,dy21,dz21);
1603 rsq22 = gmx_mm_calc_rsq_ps(dx22,dy22,dz22);
1605 rinv00 = gmx_mm_invsqrt_ps(rsq00);
1606 rinv01 = gmx_mm_invsqrt_ps(rsq01);
1607 rinv02 = gmx_mm_invsqrt_ps(rsq02);
1608 rinv10 = gmx_mm_invsqrt_ps(rsq10);
1609 rinv11 = gmx_mm_invsqrt_ps(rsq11);
1610 rinv12 = gmx_mm_invsqrt_ps(rsq12);
1611 rinv20 = gmx_mm_invsqrt_ps(rsq20);
1612 rinv21 = gmx_mm_invsqrt_ps(rsq21);
1613 rinv22 = gmx_mm_invsqrt_ps(rsq22);
1615 rinvsq00 = _mm_mul_ps(rinv00,rinv00);
1616 rinvsq01 = _mm_mul_ps(rinv01,rinv01);
1617 rinvsq02 = _mm_mul_ps(rinv02,rinv02);
1618 rinvsq10 = _mm_mul_ps(rinv10,rinv10);
1619 rinvsq11 = _mm_mul_ps(rinv11,rinv11);
1620 rinvsq12 = _mm_mul_ps(rinv12,rinv12);
1621 rinvsq20 = _mm_mul_ps(rinv20,rinv20);
1622 rinvsq21 = _mm_mul_ps(rinv21,rinv21);
1623 rinvsq22 = _mm_mul_ps(rinv22,rinv22);
1625 fjx0 = _mm_setzero_ps();
1626 fjy0 = _mm_setzero_ps();
1627 fjz0 = _mm_setzero_ps();
1628 fjx1 = _mm_setzero_ps();
1629 fjy1 = _mm_setzero_ps();
1630 fjz1 = _mm_setzero_ps();
1631 fjx2 = _mm_setzero_ps();
1632 fjy2 = _mm_setzero_ps();
1633 fjz2 = _mm_setzero_ps();
1635 /**************************
1636 * CALCULATE INTERACTIONS *
1637 **************************/
1639 r00 = _mm_mul_ps(rsq00,rinv00);
1640 r00 = _mm_andnot_ps(dummy_mask,r00);
1642 /* Calculate table index by multiplying r with table scale and truncate to integer */
1643 rt = _mm_mul_ps(r00,vftabscale);
1644 vfitab = _mm_cvttps_epi32(rt);
1645 vfeps = _mm_sub_ps(rt,_mm_cvtepi32_ps(vfitab));
1646 vfitab = _mm_slli_epi32(vfitab,3);
1648 /* COULOMB ELECTROSTATICS */
1649 velec = _mm_mul_ps(qq00,rinv00);
1650 felec = _mm_mul_ps(velec,rinvsq00);
1652 /* CUBIC SPLINE TABLE DISPERSION */
1653 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
1654 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
1655 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
1656 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
1657 _MM_TRANSPOSE4_PS(Y,F,G,H);
1658 Heps = _mm_mul_ps(vfeps,H);
1659 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
1660 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
1661 fvdw6 = _mm_mul_ps(c6_00,FF);
1663 /* CUBIC SPLINE TABLE REPULSION */
1664 vfitab = _mm_add_epi32(vfitab,ifour);
1665 Y = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,0) );
1666 F = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,1) );
1667 G = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,2) );
1668 H = _mm_load_ps( vftab + gmx_mm_extract_epi32(vfitab,3) );
1669 _MM_TRANSPOSE4_PS(Y,F,G,H);
1670 Heps = _mm_mul_ps(vfeps,H);
1671 Fp = _mm_add_ps(F,_mm_mul_ps(vfeps,_mm_add_ps(G,Heps)));
1672 FF = _mm_add_ps(Fp,_mm_mul_ps(vfeps,_mm_add_ps(G,_mm_add_ps(Heps,Heps))));
1673 fvdw12 = _mm_mul_ps(c12_00,FF);
1674 fvdw = _mm_xor_ps(signbit,_mm_mul_ps(_mm_add_ps(fvdw6,fvdw12),_mm_mul_ps(vftabscale,rinv00)));
1676 fscal = _mm_add_ps(felec,fvdw);
1678 fscal = _mm_andnot_ps(dummy_mask,fscal);
1680 /* Calculate temporary vectorial force */
1681 tx = _mm_mul_ps(fscal,dx00);
1682 ty = _mm_mul_ps(fscal,dy00);
1683 tz = _mm_mul_ps(fscal,dz00);
1685 /* Update vectorial force */
1686 fix0 = _mm_add_ps(fix0,tx);
1687 fiy0 = _mm_add_ps(fiy0,ty);
1688 fiz0 = _mm_add_ps(fiz0,tz);
1690 fjx0 = _mm_add_ps(fjx0,tx);
1691 fjy0 = _mm_add_ps(fjy0,ty);
1692 fjz0 = _mm_add_ps(fjz0,tz);
1694 /**************************
1695 * CALCULATE INTERACTIONS *
1696 **************************/
1698 /* COULOMB ELECTROSTATICS */
1699 velec = _mm_mul_ps(qq01,rinv01);
1700 felec = _mm_mul_ps(velec,rinvsq01);
1704 fscal = _mm_andnot_ps(dummy_mask,fscal);
1706 /* Calculate temporary vectorial force */
1707 tx = _mm_mul_ps(fscal,dx01);
1708 ty = _mm_mul_ps(fscal,dy01);
1709 tz = _mm_mul_ps(fscal,dz01);
1711 /* Update vectorial force */
1712 fix0 = _mm_add_ps(fix0,tx);
1713 fiy0 = _mm_add_ps(fiy0,ty);
1714 fiz0 = _mm_add_ps(fiz0,tz);
1716 fjx1 = _mm_add_ps(fjx1,tx);
1717 fjy1 = _mm_add_ps(fjy1,ty);
1718 fjz1 = _mm_add_ps(fjz1,tz);
1720 /**************************
1721 * CALCULATE INTERACTIONS *
1722 **************************/
1724 /* COULOMB ELECTROSTATICS */
1725 velec = _mm_mul_ps(qq02,rinv02);
1726 felec = _mm_mul_ps(velec,rinvsq02);
1730 fscal = _mm_andnot_ps(dummy_mask,fscal);
1732 /* Calculate temporary vectorial force */
1733 tx = _mm_mul_ps(fscal,dx02);
1734 ty = _mm_mul_ps(fscal,dy02);
1735 tz = _mm_mul_ps(fscal,dz02);
1737 /* Update vectorial force */
1738 fix0 = _mm_add_ps(fix0,tx);
1739 fiy0 = _mm_add_ps(fiy0,ty);
1740 fiz0 = _mm_add_ps(fiz0,tz);
1742 fjx2 = _mm_add_ps(fjx2,tx);
1743 fjy2 = _mm_add_ps(fjy2,ty);
1744 fjz2 = _mm_add_ps(fjz2,tz);
1746 /**************************
1747 * CALCULATE INTERACTIONS *
1748 **************************/
1750 /* COULOMB ELECTROSTATICS */
1751 velec = _mm_mul_ps(qq10,rinv10);
1752 felec = _mm_mul_ps(velec,rinvsq10);
1756 fscal = _mm_andnot_ps(dummy_mask,fscal);
1758 /* Calculate temporary vectorial force */
1759 tx = _mm_mul_ps(fscal,dx10);
1760 ty = _mm_mul_ps(fscal,dy10);
1761 tz = _mm_mul_ps(fscal,dz10);
1763 /* Update vectorial force */
1764 fix1 = _mm_add_ps(fix1,tx);
1765 fiy1 = _mm_add_ps(fiy1,ty);
1766 fiz1 = _mm_add_ps(fiz1,tz);
1768 fjx0 = _mm_add_ps(fjx0,tx);
1769 fjy0 = _mm_add_ps(fjy0,ty);
1770 fjz0 = _mm_add_ps(fjz0,tz);
1772 /**************************
1773 * CALCULATE INTERACTIONS *
1774 **************************/
1776 /* COULOMB ELECTROSTATICS */
1777 velec = _mm_mul_ps(qq11,rinv11);
1778 felec = _mm_mul_ps(velec,rinvsq11);
1782 fscal = _mm_andnot_ps(dummy_mask,fscal);
1784 /* Calculate temporary vectorial force */
1785 tx = _mm_mul_ps(fscal,dx11);
1786 ty = _mm_mul_ps(fscal,dy11);
1787 tz = _mm_mul_ps(fscal,dz11);
1789 /* Update vectorial force */
1790 fix1 = _mm_add_ps(fix1,tx);
1791 fiy1 = _mm_add_ps(fiy1,ty);
1792 fiz1 = _mm_add_ps(fiz1,tz);
1794 fjx1 = _mm_add_ps(fjx1,tx);
1795 fjy1 = _mm_add_ps(fjy1,ty);
1796 fjz1 = _mm_add_ps(fjz1,tz);
1798 /**************************
1799 * CALCULATE INTERACTIONS *
1800 **************************/
1802 /* COULOMB ELECTROSTATICS */
1803 velec = _mm_mul_ps(qq12,rinv12);
1804 felec = _mm_mul_ps(velec,rinvsq12);
1808 fscal = _mm_andnot_ps(dummy_mask,fscal);
1810 /* Calculate temporary vectorial force */
1811 tx = _mm_mul_ps(fscal,dx12);
1812 ty = _mm_mul_ps(fscal,dy12);
1813 tz = _mm_mul_ps(fscal,dz12);
1815 /* Update vectorial force */
1816 fix1 = _mm_add_ps(fix1,tx);
1817 fiy1 = _mm_add_ps(fiy1,ty);
1818 fiz1 = _mm_add_ps(fiz1,tz);
1820 fjx2 = _mm_add_ps(fjx2,tx);
1821 fjy2 = _mm_add_ps(fjy2,ty);
1822 fjz2 = _mm_add_ps(fjz2,tz);
1824 /**************************
1825 * CALCULATE INTERACTIONS *
1826 **************************/
1828 /* COULOMB ELECTROSTATICS */
1829 velec = _mm_mul_ps(qq20,rinv20);
1830 felec = _mm_mul_ps(velec,rinvsq20);
1834 fscal = _mm_andnot_ps(dummy_mask,fscal);
1836 /* Calculate temporary vectorial force */
1837 tx = _mm_mul_ps(fscal,dx20);
1838 ty = _mm_mul_ps(fscal,dy20);
1839 tz = _mm_mul_ps(fscal,dz20);
1841 /* Update vectorial force */
1842 fix2 = _mm_add_ps(fix2,tx);
1843 fiy2 = _mm_add_ps(fiy2,ty);
1844 fiz2 = _mm_add_ps(fiz2,tz);
1846 fjx0 = _mm_add_ps(fjx0,tx);
1847 fjy0 = _mm_add_ps(fjy0,ty);
1848 fjz0 = _mm_add_ps(fjz0,tz);
1850 /**************************
1851 * CALCULATE INTERACTIONS *
1852 **************************/
1854 /* COULOMB ELECTROSTATICS */
1855 velec = _mm_mul_ps(qq21,rinv21);
1856 felec = _mm_mul_ps(velec,rinvsq21);
1860 fscal = _mm_andnot_ps(dummy_mask,fscal);
1862 /* Calculate temporary vectorial force */
1863 tx = _mm_mul_ps(fscal,dx21);
1864 ty = _mm_mul_ps(fscal,dy21);
1865 tz = _mm_mul_ps(fscal,dz21);
1867 /* Update vectorial force */
1868 fix2 = _mm_add_ps(fix2,tx);
1869 fiy2 = _mm_add_ps(fiy2,ty);
1870 fiz2 = _mm_add_ps(fiz2,tz);
1872 fjx1 = _mm_add_ps(fjx1,tx);
1873 fjy1 = _mm_add_ps(fjy1,ty);
1874 fjz1 = _mm_add_ps(fjz1,tz);
1876 /**************************
1877 * CALCULATE INTERACTIONS *
1878 **************************/
1880 /* COULOMB ELECTROSTATICS */
1881 velec = _mm_mul_ps(qq22,rinv22);
1882 felec = _mm_mul_ps(velec,rinvsq22);
1886 fscal = _mm_andnot_ps(dummy_mask,fscal);
1888 /* Calculate temporary vectorial force */
1889 tx = _mm_mul_ps(fscal,dx22);
1890 ty = _mm_mul_ps(fscal,dy22);
1891 tz = _mm_mul_ps(fscal,dz22);
1893 /* Update vectorial force */
1894 fix2 = _mm_add_ps(fix2,tx);
1895 fiy2 = _mm_add_ps(fiy2,ty);
1896 fiz2 = _mm_add_ps(fiz2,tz);
1898 fjx2 = _mm_add_ps(fjx2,tx);
1899 fjy2 = _mm_add_ps(fjy2,ty);
1900 fjz2 = _mm_add_ps(fjz2,tz);
1902 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1903 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1904 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1905 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1907 gmx_mm_decrement_3rvec_4ptr_swizzle_ps(fjptrA,fjptrB,fjptrC,fjptrD,
1908 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,fjx2,fjy2,fjz2);
1910 /* Inner loop uses 271 flops */
1913 /* End of innermost loop */
1915 gmx_mm_update_iforce_3atom_swizzle_ps(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,
1916 f+i_coord_offset,fshift+i_shift_offset);
1918 /* Increment number of inner iterations */
1919 inneriter += j_index_end - j_index_start;
1921 /* Outer loop uses 18 flops */
1924 /* Increment number of outer iterations */
1927 /* Update outer/inner flops */
1929 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W3W3_F,outeriter*18 + inneriter*271);