2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_256_double kernel generator.
42 #include "../nb_kernel.h"
43 #include "types/simple.h"
44 #include "gromacs/math/vec.h"
47 #include "gromacs/simd/math_x86_avx_256_double.h"
48 #include "kernelutil_x86_avx_256_double.h"
51 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwLJ_GeomW4W4_VF_avx_256_double
52 * Electrostatics interaction: Coulomb
53 * VdW interaction: LennardJones
54 * Geometry: Water4-Water4
55 * Calculate force/pot: PotentialAndForce
58 nb_kernel_ElecCoul_VdwLJ_GeomW4W4_VF_avx_256_double
59 (t_nblist * gmx_restrict nlist,
60 rvec * gmx_restrict xx,
61 rvec * gmx_restrict ff,
62 t_forcerec * gmx_restrict fr,
63 t_mdatoms * gmx_restrict mdatoms,
64 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
65 t_nrnb * gmx_restrict nrnb)
67 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
68 * just 0 for non-waters.
69 * Suffixes A,B,C,D refer to j loop unrolling done with AVX, e.g. for the four different
70 * jnr indices corresponding to data put in the four positions in the SIMD register.
72 int i_shift_offset,i_coord_offset,outeriter,inneriter;
73 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
74 int jnrA,jnrB,jnrC,jnrD;
75 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
76 int jnrlistE,jnrlistF,jnrlistG,jnrlistH;
77 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
78 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
80 real *shiftvec,*fshift,*x,*f;
81 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
83 __m256d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
84 real * vdwioffsetptr0;
85 __m256d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
86 real * vdwioffsetptr1;
87 __m256d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
88 real * vdwioffsetptr2;
89 __m256d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
90 real * vdwioffsetptr3;
91 __m256d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
92 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
93 __m256d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
94 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
95 __m256d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
96 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
97 __m256d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
98 int vdwjidx3A,vdwjidx3B,vdwjidx3C,vdwjidx3D;
99 __m256d jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
100 __m256d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
101 __m256d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
102 __m256d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
103 __m256d dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
104 __m256d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
105 __m256d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
106 __m256d dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
107 __m256d dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
108 __m256d dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
109 __m256d dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
110 __m256d velec,felec,velecsum,facel,crf,krf,krf2;
113 __m256d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
116 __m256d one_sixth = _mm256_set1_pd(1.0/6.0);
117 __m256d one_twelfth = _mm256_set1_pd(1.0/12.0);
118 __m256d dummy_mask,cutoff_mask;
119 __m128 tmpmask0,tmpmask1;
120 __m256d signbit = _mm256_castsi256_pd( _mm256_set1_epi32(0x80000000) );
121 __m256d one = _mm256_set1_pd(1.0);
122 __m256d two = _mm256_set1_pd(2.0);
128 jindex = nlist->jindex;
130 shiftidx = nlist->shift;
132 shiftvec = fr->shift_vec[0];
133 fshift = fr->fshift[0];
134 facel = _mm256_set1_pd(fr->epsfac);
135 charge = mdatoms->chargeA;
136 nvdwtype = fr->ntype;
138 vdwtype = mdatoms->typeA;
140 /* Setup water-specific parameters */
141 inr = nlist->iinr[0];
142 iq1 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+1]));
143 iq2 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+2]));
144 iq3 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+3]));
145 vdwioffsetptr0 = vdwparam+2*nvdwtype*vdwtype[inr+0];
147 jq1 = _mm256_set1_pd(charge[inr+1]);
148 jq2 = _mm256_set1_pd(charge[inr+2]);
149 jq3 = _mm256_set1_pd(charge[inr+3]);
150 vdwjidx0A = 2*vdwtype[inr+0];
151 c6_00 = _mm256_set1_pd(vdwioffsetptr0[vdwjidx0A]);
152 c12_00 = _mm256_set1_pd(vdwioffsetptr0[vdwjidx0A+1]);
153 qq11 = _mm256_mul_pd(iq1,jq1);
154 qq12 = _mm256_mul_pd(iq1,jq2);
155 qq13 = _mm256_mul_pd(iq1,jq3);
156 qq21 = _mm256_mul_pd(iq2,jq1);
157 qq22 = _mm256_mul_pd(iq2,jq2);
158 qq23 = _mm256_mul_pd(iq2,jq3);
159 qq31 = _mm256_mul_pd(iq3,jq1);
160 qq32 = _mm256_mul_pd(iq3,jq2);
161 qq33 = _mm256_mul_pd(iq3,jq3);
163 /* Avoid stupid compiler warnings */
164 jnrA = jnrB = jnrC = jnrD = 0;
173 for(iidx=0;iidx<4*DIM;iidx++)
178 /* Start outer loop over neighborlists */
179 for(iidx=0; iidx<nri; iidx++)
181 /* Load shift vector for this list */
182 i_shift_offset = DIM*shiftidx[iidx];
184 /* Load limits for loop over neighbors */
185 j_index_start = jindex[iidx];
186 j_index_end = jindex[iidx+1];
188 /* Get outer coordinate index */
190 i_coord_offset = DIM*inr;
192 /* Load i particle coords and add shift vector */
193 gmx_mm256_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
194 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
196 fix0 = _mm256_setzero_pd();
197 fiy0 = _mm256_setzero_pd();
198 fiz0 = _mm256_setzero_pd();
199 fix1 = _mm256_setzero_pd();
200 fiy1 = _mm256_setzero_pd();
201 fiz1 = _mm256_setzero_pd();
202 fix2 = _mm256_setzero_pd();
203 fiy2 = _mm256_setzero_pd();
204 fiz2 = _mm256_setzero_pd();
205 fix3 = _mm256_setzero_pd();
206 fiy3 = _mm256_setzero_pd();
207 fiz3 = _mm256_setzero_pd();
209 /* Reset potential sums */
210 velecsum = _mm256_setzero_pd();
211 vvdwsum = _mm256_setzero_pd();
213 /* Start inner kernel loop */
214 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
217 /* Get j neighbor index, and coordinate index */
222 j_coord_offsetA = DIM*jnrA;
223 j_coord_offsetB = DIM*jnrB;
224 j_coord_offsetC = DIM*jnrC;
225 j_coord_offsetD = DIM*jnrD;
227 /* load j atom coordinates */
228 gmx_mm256_load_4rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
229 x+j_coord_offsetC,x+j_coord_offsetD,
230 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
231 &jy2,&jz2,&jx3,&jy3,&jz3);
233 /* Calculate displacement vector */
234 dx00 = _mm256_sub_pd(ix0,jx0);
235 dy00 = _mm256_sub_pd(iy0,jy0);
236 dz00 = _mm256_sub_pd(iz0,jz0);
237 dx11 = _mm256_sub_pd(ix1,jx1);
238 dy11 = _mm256_sub_pd(iy1,jy1);
239 dz11 = _mm256_sub_pd(iz1,jz1);
240 dx12 = _mm256_sub_pd(ix1,jx2);
241 dy12 = _mm256_sub_pd(iy1,jy2);
242 dz12 = _mm256_sub_pd(iz1,jz2);
243 dx13 = _mm256_sub_pd(ix1,jx3);
244 dy13 = _mm256_sub_pd(iy1,jy3);
245 dz13 = _mm256_sub_pd(iz1,jz3);
246 dx21 = _mm256_sub_pd(ix2,jx1);
247 dy21 = _mm256_sub_pd(iy2,jy1);
248 dz21 = _mm256_sub_pd(iz2,jz1);
249 dx22 = _mm256_sub_pd(ix2,jx2);
250 dy22 = _mm256_sub_pd(iy2,jy2);
251 dz22 = _mm256_sub_pd(iz2,jz2);
252 dx23 = _mm256_sub_pd(ix2,jx3);
253 dy23 = _mm256_sub_pd(iy2,jy3);
254 dz23 = _mm256_sub_pd(iz2,jz3);
255 dx31 = _mm256_sub_pd(ix3,jx1);
256 dy31 = _mm256_sub_pd(iy3,jy1);
257 dz31 = _mm256_sub_pd(iz3,jz1);
258 dx32 = _mm256_sub_pd(ix3,jx2);
259 dy32 = _mm256_sub_pd(iy3,jy2);
260 dz32 = _mm256_sub_pd(iz3,jz2);
261 dx33 = _mm256_sub_pd(ix3,jx3);
262 dy33 = _mm256_sub_pd(iy3,jy3);
263 dz33 = _mm256_sub_pd(iz3,jz3);
265 /* Calculate squared distance and things based on it */
266 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
267 rsq11 = gmx_mm256_calc_rsq_pd(dx11,dy11,dz11);
268 rsq12 = gmx_mm256_calc_rsq_pd(dx12,dy12,dz12);
269 rsq13 = gmx_mm256_calc_rsq_pd(dx13,dy13,dz13);
270 rsq21 = gmx_mm256_calc_rsq_pd(dx21,dy21,dz21);
271 rsq22 = gmx_mm256_calc_rsq_pd(dx22,dy22,dz22);
272 rsq23 = gmx_mm256_calc_rsq_pd(dx23,dy23,dz23);
273 rsq31 = gmx_mm256_calc_rsq_pd(dx31,dy31,dz31);
274 rsq32 = gmx_mm256_calc_rsq_pd(dx32,dy32,dz32);
275 rsq33 = gmx_mm256_calc_rsq_pd(dx33,dy33,dz33);
277 rinv11 = gmx_mm256_invsqrt_pd(rsq11);
278 rinv12 = gmx_mm256_invsqrt_pd(rsq12);
279 rinv13 = gmx_mm256_invsqrt_pd(rsq13);
280 rinv21 = gmx_mm256_invsqrt_pd(rsq21);
281 rinv22 = gmx_mm256_invsqrt_pd(rsq22);
282 rinv23 = gmx_mm256_invsqrt_pd(rsq23);
283 rinv31 = gmx_mm256_invsqrt_pd(rsq31);
284 rinv32 = gmx_mm256_invsqrt_pd(rsq32);
285 rinv33 = gmx_mm256_invsqrt_pd(rsq33);
287 rinvsq00 = gmx_mm256_inv_pd(rsq00);
288 rinvsq11 = _mm256_mul_pd(rinv11,rinv11);
289 rinvsq12 = _mm256_mul_pd(rinv12,rinv12);
290 rinvsq13 = _mm256_mul_pd(rinv13,rinv13);
291 rinvsq21 = _mm256_mul_pd(rinv21,rinv21);
292 rinvsq22 = _mm256_mul_pd(rinv22,rinv22);
293 rinvsq23 = _mm256_mul_pd(rinv23,rinv23);
294 rinvsq31 = _mm256_mul_pd(rinv31,rinv31);
295 rinvsq32 = _mm256_mul_pd(rinv32,rinv32);
296 rinvsq33 = _mm256_mul_pd(rinv33,rinv33);
298 fjx0 = _mm256_setzero_pd();
299 fjy0 = _mm256_setzero_pd();
300 fjz0 = _mm256_setzero_pd();
301 fjx1 = _mm256_setzero_pd();
302 fjy1 = _mm256_setzero_pd();
303 fjz1 = _mm256_setzero_pd();
304 fjx2 = _mm256_setzero_pd();
305 fjy2 = _mm256_setzero_pd();
306 fjz2 = _mm256_setzero_pd();
307 fjx3 = _mm256_setzero_pd();
308 fjy3 = _mm256_setzero_pd();
309 fjz3 = _mm256_setzero_pd();
311 /**************************
312 * CALCULATE INTERACTIONS *
313 **************************/
315 /* LENNARD-JONES DISPERSION/REPULSION */
317 rinvsix = _mm256_mul_pd(_mm256_mul_pd(rinvsq00,rinvsq00),rinvsq00);
318 vvdw6 = _mm256_mul_pd(c6_00,rinvsix);
319 vvdw12 = _mm256_mul_pd(c12_00,_mm256_mul_pd(rinvsix,rinvsix));
320 vvdw = _mm256_sub_pd( _mm256_mul_pd(vvdw12,one_twelfth) , _mm256_mul_pd(vvdw6,one_sixth) );
321 fvdw = _mm256_mul_pd(_mm256_sub_pd(vvdw12,vvdw6),rinvsq00);
323 /* Update potential sum for this i atom from the interaction with this j atom. */
324 vvdwsum = _mm256_add_pd(vvdwsum,vvdw);
328 /* Calculate temporary vectorial force */
329 tx = _mm256_mul_pd(fscal,dx00);
330 ty = _mm256_mul_pd(fscal,dy00);
331 tz = _mm256_mul_pd(fscal,dz00);
333 /* Update vectorial force */
334 fix0 = _mm256_add_pd(fix0,tx);
335 fiy0 = _mm256_add_pd(fiy0,ty);
336 fiz0 = _mm256_add_pd(fiz0,tz);
338 fjx0 = _mm256_add_pd(fjx0,tx);
339 fjy0 = _mm256_add_pd(fjy0,ty);
340 fjz0 = _mm256_add_pd(fjz0,tz);
342 /**************************
343 * CALCULATE INTERACTIONS *
344 **************************/
346 /* COULOMB ELECTROSTATICS */
347 velec = _mm256_mul_pd(qq11,rinv11);
348 felec = _mm256_mul_pd(velec,rinvsq11);
350 /* Update potential sum for this i atom from the interaction with this j atom. */
351 velecsum = _mm256_add_pd(velecsum,velec);
355 /* Calculate temporary vectorial force */
356 tx = _mm256_mul_pd(fscal,dx11);
357 ty = _mm256_mul_pd(fscal,dy11);
358 tz = _mm256_mul_pd(fscal,dz11);
360 /* Update vectorial force */
361 fix1 = _mm256_add_pd(fix1,tx);
362 fiy1 = _mm256_add_pd(fiy1,ty);
363 fiz1 = _mm256_add_pd(fiz1,tz);
365 fjx1 = _mm256_add_pd(fjx1,tx);
366 fjy1 = _mm256_add_pd(fjy1,ty);
367 fjz1 = _mm256_add_pd(fjz1,tz);
369 /**************************
370 * CALCULATE INTERACTIONS *
371 **************************/
373 /* COULOMB ELECTROSTATICS */
374 velec = _mm256_mul_pd(qq12,rinv12);
375 felec = _mm256_mul_pd(velec,rinvsq12);
377 /* Update potential sum for this i atom from the interaction with this j atom. */
378 velecsum = _mm256_add_pd(velecsum,velec);
382 /* Calculate temporary vectorial force */
383 tx = _mm256_mul_pd(fscal,dx12);
384 ty = _mm256_mul_pd(fscal,dy12);
385 tz = _mm256_mul_pd(fscal,dz12);
387 /* Update vectorial force */
388 fix1 = _mm256_add_pd(fix1,tx);
389 fiy1 = _mm256_add_pd(fiy1,ty);
390 fiz1 = _mm256_add_pd(fiz1,tz);
392 fjx2 = _mm256_add_pd(fjx2,tx);
393 fjy2 = _mm256_add_pd(fjy2,ty);
394 fjz2 = _mm256_add_pd(fjz2,tz);
396 /**************************
397 * CALCULATE INTERACTIONS *
398 **************************/
400 /* COULOMB ELECTROSTATICS */
401 velec = _mm256_mul_pd(qq13,rinv13);
402 felec = _mm256_mul_pd(velec,rinvsq13);
404 /* Update potential sum for this i atom from the interaction with this j atom. */
405 velecsum = _mm256_add_pd(velecsum,velec);
409 /* Calculate temporary vectorial force */
410 tx = _mm256_mul_pd(fscal,dx13);
411 ty = _mm256_mul_pd(fscal,dy13);
412 tz = _mm256_mul_pd(fscal,dz13);
414 /* Update vectorial force */
415 fix1 = _mm256_add_pd(fix1,tx);
416 fiy1 = _mm256_add_pd(fiy1,ty);
417 fiz1 = _mm256_add_pd(fiz1,tz);
419 fjx3 = _mm256_add_pd(fjx3,tx);
420 fjy3 = _mm256_add_pd(fjy3,ty);
421 fjz3 = _mm256_add_pd(fjz3,tz);
423 /**************************
424 * CALCULATE INTERACTIONS *
425 **************************/
427 /* COULOMB ELECTROSTATICS */
428 velec = _mm256_mul_pd(qq21,rinv21);
429 felec = _mm256_mul_pd(velec,rinvsq21);
431 /* Update potential sum for this i atom from the interaction with this j atom. */
432 velecsum = _mm256_add_pd(velecsum,velec);
436 /* Calculate temporary vectorial force */
437 tx = _mm256_mul_pd(fscal,dx21);
438 ty = _mm256_mul_pd(fscal,dy21);
439 tz = _mm256_mul_pd(fscal,dz21);
441 /* Update vectorial force */
442 fix2 = _mm256_add_pd(fix2,tx);
443 fiy2 = _mm256_add_pd(fiy2,ty);
444 fiz2 = _mm256_add_pd(fiz2,tz);
446 fjx1 = _mm256_add_pd(fjx1,tx);
447 fjy1 = _mm256_add_pd(fjy1,ty);
448 fjz1 = _mm256_add_pd(fjz1,tz);
450 /**************************
451 * CALCULATE INTERACTIONS *
452 **************************/
454 /* COULOMB ELECTROSTATICS */
455 velec = _mm256_mul_pd(qq22,rinv22);
456 felec = _mm256_mul_pd(velec,rinvsq22);
458 /* Update potential sum for this i atom from the interaction with this j atom. */
459 velecsum = _mm256_add_pd(velecsum,velec);
463 /* Calculate temporary vectorial force */
464 tx = _mm256_mul_pd(fscal,dx22);
465 ty = _mm256_mul_pd(fscal,dy22);
466 tz = _mm256_mul_pd(fscal,dz22);
468 /* Update vectorial force */
469 fix2 = _mm256_add_pd(fix2,tx);
470 fiy2 = _mm256_add_pd(fiy2,ty);
471 fiz2 = _mm256_add_pd(fiz2,tz);
473 fjx2 = _mm256_add_pd(fjx2,tx);
474 fjy2 = _mm256_add_pd(fjy2,ty);
475 fjz2 = _mm256_add_pd(fjz2,tz);
477 /**************************
478 * CALCULATE INTERACTIONS *
479 **************************/
481 /* COULOMB ELECTROSTATICS */
482 velec = _mm256_mul_pd(qq23,rinv23);
483 felec = _mm256_mul_pd(velec,rinvsq23);
485 /* Update potential sum for this i atom from the interaction with this j atom. */
486 velecsum = _mm256_add_pd(velecsum,velec);
490 /* Calculate temporary vectorial force */
491 tx = _mm256_mul_pd(fscal,dx23);
492 ty = _mm256_mul_pd(fscal,dy23);
493 tz = _mm256_mul_pd(fscal,dz23);
495 /* Update vectorial force */
496 fix2 = _mm256_add_pd(fix2,tx);
497 fiy2 = _mm256_add_pd(fiy2,ty);
498 fiz2 = _mm256_add_pd(fiz2,tz);
500 fjx3 = _mm256_add_pd(fjx3,tx);
501 fjy3 = _mm256_add_pd(fjy3,ty);
502 fjz3 = _mm256_add_pd(fjz3,tz);
504 /**************************
505 * CALCULATE INTERACTIONS *
506 **************************/
508 /* COULOMB ELECTROSTATICS */
509 velec = _mm256_mul_pd(qq31,rinv31);
510 felec = _mm256_mul_pd(velec,rinvsq31);
512 /* Update potential sum for this i atom from the interaction with this j atom. */
513 velecsum = _mm256_add_pd(velecsum,velec);
517 /* Calculate temporary vectorial force */
518 tx = _mm256_mul_pd(fscal,dx31);
519 ty = _mm256_mul_pd(fscal,dy31);
520 tz = _mm256_mul_pd(fscal,dz31);
522 /* Update vectorial force */
523 fix3 = _mm256_add_pd(fix3,tx);
524 fiy3 = _mm256_add_pd(fiy3,ty);
525 fiz3 = _mm256_add_pd(fiz3,tz);
527 fjx1 = _mm256_add_pd(fjx1,tx);
528 fjy1 = _mm256_add_pd(fjy1,ty);
529 fjz1 = _mm256_add_pd(fjz1,tz);
531 /**************************
532 * CALCULATE INTERACTIONS *
533 **************************/
535 /* COULOMB ELECTROSTATICS */
536 velec = _mm256_mul_pd(qq32,rinv32);
537 felec = _mm256_mul_pd(velec,rinvsq32);
539 /* Update potential sum for this i atom from the interaction with this j atom. */
540 velecsum = _mm256_add_pd(velecsum,velec);
544 /* Calculate temporary vectorial force */
545 tx = _mm256_mul_pd(fscal,dx32);
546 ty = _mm256_mul_pd(fscal,dy32);
547 tz = _mm256_mul_pd(fscal,dz32);
549 /* Update vectorial force */
550 fix3 = _mm256_add_pd(fix3,tx);
551 fiy3 = _mm256_add_pd(fiy3,ty);
552 fiz3 = _mm256_add_pd(fiz3,tz);
554 fjx2 = _mm256_add_pd(fjx2,tx);
555 fjy2 = _mm256_add_pd(fjy2,ty);
556 fjz2 = _mm256_add_pd(fjz2,tz);
558 /**************************
559 * CALCULATE INTERACTIONS *
560 **************************/
562 /* COULOMB ELECTROSTATICS */
563 velec = _mm256_mul_pd(qq33,rinv33);
564 felec = _mm256_mul_pd(velec,rinvsq33);
566 /* Update potential sum for this i atom from the interaction with this j atom. */
567 velecsum = _mm256_add_pd(velecsum,velec);
571 /* Calculate temporary vectorial force */
572 tx = _mm256_mul_pd(fscal,dx33);
573 ty = _mm256_mul_pd(fscal,dy33);
574 tz = _mm256_mul_pd(fscal,dz33);
576 /* Update vectorial force */
577 fix3 = _mm256_add_pd(fix3,tx);
578 fiy3 = _mm256_add_pd(fiy3,ty);
579 fiz3 = _mm256_add_pd(fiz3,tz);
581 fjx3 = _mm256_add_pd(fjx3,tx);
582 fjy3 = _mm256_add_pd(fjy3,ty);
583 fjz3 = _mm256_add_pd(fjz3,tz);
585 fjptrA = f+j_coord_offsetA;
586 fjptrB = f+j_coord_offsetB;
587 fjptrC = f+j_coord_offsetC;
588 fjptrD = f+j_coord_offsetD;
590 gmx_mm256_decrement_4rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,
591 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
592 fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
594 /* Inner loop uses 278 flops */
600 /* Get j neighbor index, and coordinate index */
601 jnrlistA = jjnr[jidx];
602 jnrlistB = jjnr[jidx+1];
603 jnrlistC = jjnr[jidx+2];
604 jnrlistD = jjnr[jidx+3];
605 /* Sign of each element will be negative for non-real atoms.
606 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
607 * so use it as val = _mm_andnot_pd(mask,val) to clear dummy entries.
609 tmpmask0 = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
611 tmpmask1 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(3,3,2,2));
612 tmpmask0 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(1,1,0,0));
613 dummy_mask = _mm256_castps_pd(gmx_mm256_set_m128(tmpmask1,tmpmask0));
615 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
616 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
617 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
618 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
619 j_coord_offsetA = DIM*jnrA;
620 j_coord_offsetB = DIM*jnrB;
621 j_coord_offsetC = DIM*jnrC;
622 j_coord_offsetD = DIM*jnrD;
624 /* load j atom coordinates */
625 gmx_mm256_load_4rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
626 x+j_coord_offsetC,x+j_coord_offsetD,
627 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
628 &jy2,&jz2,&jx3,&jy3,&jz3);
630 /* Calculate displacement vector */
631 dx00 = _mm256_sub_pd(ix0,jx0);
632 dy00 = _mm256_sub_pd(iy0,jy0);
633 dz00 = _mm256_sub_pd(iz0,jz0);
634 dx11 = _mm256_sub_pd(ix1,jx1);
635 dy11 = _mm256_sub_pd(iy1,jy1);
636 dz11 = _mm256_sub_pd(iz1,jz1);
637 dx12 = _mm256_sub_pd(ix1,jx2);
638 dy12 = _mm256_sub_pd(iy1,jy2);
639 dz12 = _mm256_sub_pd(iz1,jz2);
640 dx13 = _mm256_sub_pd(ix1,jx3);
641 dy13 = _mm256_sub_pd(iy1,jy3);
642 dz13 = _mm256_sub_pd(iz1,jz3);
643 dx21 = _mm256_sub_pd(ix2,jx1);
644 dy21 = _mm256_sub_pd(iy2,jy1);
645 dz21 = _mm256_sub_pd(iz2,jz1);
646 dx22 = _mm256_sub_pd(ix2,jx2);
647 dy22 = _mm256_sub_pd(iy2,jy2);
648 dz22 = _mm256_sub_pd(iz2,jz2);
649 dx23 = _mm256_sub_pd(ix2,jx3);
650 dy23 = _mm256_sub_pd(iy2,jy3);
651 dz23 = _mm256_sub_pd(iz2,jz3);
652 dx31 = _mm256_sub_pd(ix3,jx1);
653 dy31 = _mm256_sub_pd(iy3,jy1);
654 dz31 = _mm256_sub_pd(iz3,jz1);
655 dx32 = _mm256_sub_pd(ix3,jx2);
656 dy32 = _mm256_sub_pd(iy3,jy2);
657 dz32 = _mm256_sub_pd(iz3,jz2);
658 dx33 = _mm256_sub_pd(ix3,jx3);
659 dy33 = _mm256_sub_pd(iy3,jy3);
660 dz33 = _mm256_sub_pd(iz3,jz3);
662 /* Calculate squared distance and things based on it */
663 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
664 rsq11 = gmx_mm256_calc_rsq_pd(dx11,dy11,dz11);
665 rsq12 = gmx_mm256_calc_rsq_pd(dx12,dy12,dz12);
666 rsq13 = gmx_mm256_calc_rsq_pd(dx13,dy13,dz13);
667 rsq21 = gmx_mm256_calc_rsq_pd(dx21,dy21,dz21);
668 rsq22 = gmx_mm256_calc_rsq_pd(dx22,dy22,dz22);
669 rsq23 = gmx_mm256_calc_rsq_pd(dx23,dy23,dz23);
670 rsq31 = gmx_mm256_calc_rsq_pd(dx31,dy31,dz31);
671 rsq32 = gmx_mm256_calc_rsq_pd(dx32,dy32,dz32);
672 rsq33 = gmx_mm256_calc_rsq_pd(dx33,dy33,dz33);
674 rinv11 = gmx_mm256_invsqrt_pd(rsq11);
675 rinv12 = gmx_mm256_invsqrt_pd(rsq12);
676 rinv13 = gmx_mm256_invsqrt_pd(rsq13);
677 rinv21 = gmx_mm256_invsqrt_pd(rsq21);
678 rinv22 = gmx_mm256_invsqrt_pd(rsq22);
679 rinv23 = gmx_mm256_invsqrt_pd(rsq23);
680 rinv31 = gmx_mm256_invsqrt_pd(rsq31);
681 rinv32 = gmx_mm256_invsqrt_pd(rsq32);
682 rinv33 = gmx_mm256_invsqrt_pd(rsq33);
684 rinvsq00 = gmx_mm256_inv_pd(rsq00);
685 rinvsq11 = _mm256_mul_pd(rinv11,rinv11);
686 rinvsq12 = _mm256_mul_pd(rinv12,rinv12);
687 rinvsq13 = _mm256_mul_pd(rinv13,rinv13);
688 rinvsq21 = _mm256_mul_pd(rinv21,rinv21);
689 rinvsq22 = _mm256_mul_pd(rinv22,rinv22);
690 rinvsq23 = _mm256_mul_pd(rinv23,rinv23);
691 rinvsq31 = _mm256_mul_pd(rinv31,rinv31);
692 rinvsq32 = _mm256_mul_pd(rinv32,rinv32);
693 rinvsq33 = _mm256_mul_pd(rinv33,rinv33);
695 fjx0 = _mm256_setzero_pd();
696 fjy0 = _mm256_setzero_pd();
697 fjz0 = _mm256_setzero_pd();
698 fjx1 = _mm256_setzero_pd();
699 fjy1 = _mm256_setzero_pd();
700 fjz1 = _mm256_setzero_pd();
701 fjx2 = _mm256_setzero_pd();
702 fjy2 = _mm256_setzero_pd();
703 fjz2 = _mm256_setzero_pd();
704 fjx3 = _mm256_setzero_pd();
705 fjy3 = _mm256_setzero_pd();
706 fjz3 = _mm256_setzero_pd();
708 /**************************
709 * CALCULATE INTERACTIONS *
710 **************************/
712 /* LENNARD-JONES DISPERSION/REPULSION */
714 rinvsix = _mm256_mul_pd(_mm256_mul_pd(rinvsq00,rinvsq00),rinvsq00);
715 vvdw6 = _mm256_mul_pd(c6_00,rinvsix);
716 vvdw12 = _mm256_mul_pd(c12_00,_mm256_mul_pd(rinvsix,rinvsix));
717 vvdw = _mm256_sub_pd( _mm256_mul_pd(vvdw12,one_twelfth) , _mm256_mul_pd(vvdw6,one_sixth) );
718 fvdw = _mm256_mul_pd(_mm256_sub_pd(vvdw12,vvdw6),rinvsq00);
720 /* Update potential sum for this i atom from the interaction with this j atom. */
721 vvdw = _mm256_andnot_pd(dummy_mask,vvdw);
722 vvdwsum = _mm256_add_pd(vvdwsum,vvdw);
726 fscal = _mm256_andnot_pd(dummy_mask,fscal);
728 /* Calculate temporary vectorial force */
729 tx = _mm256_mul_pd(fscal,dx00);
730 ty = _mm256_mul_pd(fscal,dy00);
731 tz = _mm256_mul_pd(fscal,dz00);
733 /* Update vectorial force */
734 fix0 = _mm256_add_pd(fix0,tx);
735 fiy0 = _mm256_add_pd(fiy0,ty);
736 fiz0 = _mm256_add_pd(fiz0,tz);
738 fjx0 = _mm256_add_pd(fjx0,tx);
739 fjy0 = _mm256_add_pd(fjy0,ty);
740 fjz0 = _mm256_add_pd(fjz0,tz);
742 /**************************
743 * CALCULATE INTERACTIONS *
744 **************************/
746 /* COULOMB ELECTROSTATICS */
747 velec = _mm256_mul_pd(qq11,rinv11);
748 felec = _mm256_mul_pd(velec,rinvsq11);
750 /* Update potential sum for this i atom from the interaction with this j atom. */
751 velec = _mm256_andnot_pd(dummy_mask,velec);
752 velecsum = _mm256_add_pd(velecsum,velec);
756 fscal = _mm256_andnot_pd(dummy_mask,fscal);
758 /* Calculate temporary vectorial force */
759 tx = _mm256_mul_pd(fscal,dx11);
760 ty = _mm256_mul_pd(fscal,dy11);
761 tz = _mm256_mul_pd(fscal,dz11);
763 /* Update vectorial force */
764 fix1 = _mm256_add_pd(fix1,tx);
765 fiy1 = _mm256_add_pd(fiy1,ty);
766 fiz1 = _mm256_add_pd(fiz1,tz);
768 fjx1 = _mm256_add_pd(fjx1,tx);
769 fjy1 = _mm256_add_pd(fjy1,ty);
770 fjz1 = _mm256_add_pd(fjz1,tz);
772 /**************************
773 * CALCULATE INTERACTIONS *
774 **************************/
776 /* COULOMB ELECTROSTATICS */
777 velec = _mm256_mul_pd(qq12,rinv12);
778 felec = _mm256_mul_pd(velec,rinvsq12);
780 /* Update potential sum for this i atom from the interaction with this j atom. */
781 velec = _mm256_andnot_pd(dummy_mask,velec);
782 velecsum = _mm256_add_pd(velecsum,velec);
786 fscal = _mm256_andnot_pd(dummy_mask,fscal);
788 /* Calculate temporary vectorial force */
789 tx = _mm256_mul_pd(fscal,dx12);
790 ty = _mm256_mul_pd(fscal,dy12);
791 tz = _mm256_mul_pd(fscal,dz12);
793 /* Update vectorial force */
794 fix1 = _mm256_add_pd(fix1,tx);
795 fiy1 = _mm256_add_pd(fiy1,ty);
796 fiz1 = _mm256_add_pd(fiz1,tz);
798 fjx2 = _mm256_add_pd(fjx2,tx);
799 fjy2 = _mm256_add_pd(fjy2,ty);
800 fjz2 = _mm256_add_pd(fjz2,tz);
802 /**************************
803 * CALCULATE INTERACTIONS *
804 **************************/
806 /* COULOMB ELECTROSTATICS */
807 velec = _mm256_mul_pd(qq13,rinv13);
808 felec = _mm256_mul_pd(velec,rinvsq13);
810 /* Update potential sum for this i atom from the interaction with this j atom. */
811 velec = _mm256_andnot_pd(dummy_mask,velec);
812 velecsum = _mm256_add_pd(velecsum,velec);
816 fscal = _mm256_andnot_pd(dummy_mask,fscal);
818 /* Calculate temporary vectorial force */
819 tx = _mm256_mul_pd(fscal,dx13);
820 ty = _mm256_mul_pd(fscal,dy13);
821 tz = _mm256_mul_pd(fscal,dz13);
823 /* Update vectorial force */
824 fix1 = _mm256_add_pd(fix1,tx);
825 fiy1 = _mm256_add_pd(fiy1,ty);
826 fiz1 = _mm256_add_pd(fiz1,tz);
828 fjx3 = _mm256_add_pd(fjx3,tx);
829 fjy3 = _mm256_add_pd(fjy3,ty);
830 fjz3 = _mm256_add_pd(fjz3,tz);
832 /**************************
833 * CALCULATE INTERACTIONS *
834 **************************/
836 /* COULOMB ELECTROSTATICS */
837 velec = _mm256_mul_pd(qq21,rinv21);
838 felec = _mm256_mul_pd(velec,rinvsq21);
840 /* Update potential sum for this i atom from the interaction with this j atom. */
841 velec = _mm256_andnot_pd(dummy_mask,velec);
842 velecsum = _mm256_add_pd(velecsum,velec);
846 fscal = _mm256_andnot_pd(dummy_mask,fscal);
848 /* Calculate temporary vectorial force */
849 tx = _mm256_mul_pd(fscal,dx21);
850 ty = _mm256_mul_pd(fscal,dy21);
851 tz = _mm256_mul_pd(fscal,dz21);
853 /* Update vectorial force */
854 fix2 = _mm256_add_pd(fix2,tx);
855 fiy2 = _mm256_add_pd(fiy2,ty);
856 fiz2 = _mm256_add_pd(fiz2,tz);
858 fjx1 = _mm256_add_pd(fjx1,tx);
859 fjy1 = _mm256_add_pd(fjy1,ty);
860 fjz1 = _mm256_add_pd(fjz1,tz);
862 /**************************
863 * CALCULATE INTERACTIONS *
864 **************************/
866 /* COULOMB ELECTROSTATICS */
867 velec = _mm256_mul_pd(qq22,rinv22);
868 felec = _mm256_mul_pd(velec,rinvsq22);
870 /* Update potential sum for this i atom from the interaction with this j atom. */
871 velec = _mm256_andnot_pd(dummy_mask,velec);
872 velecsum = _mm256_add_pd(velecsum,velec);
876 fscal = _mm256_andnot_pd(dummy_mask,fscal);
878 /* Calculate temporary vectorial force */
879 tx = _mm256_mul_pd(fscal,dx22);
880 ty = _mm256_mul_pd(fscal,dy22);
881 tz = _mm256_mul_pd(fscal,dz22);
883 /* Update vectorial force */
884 fix2 = _mm256_add_pd(fix2,tx);
885 fiy2 = _mm256_add_pd(fiy2,ty);
886 fiz2 = _mm256_add_pd(fiz2,tz);
888 fjx2 = _mm256_add_pd(fjx2,tx);
889 fjy2 = _mm256_add_pd(fjy2,ty);
890 fjz2 = _mm256_add_pd(fjz2,tz);
892 /**************************
893 * CALCULATE INTERACTIONS *
894 **************************/
896 /* COULOMB ELECTROSTATICS */
897 velec = _mm256_mul_pd(qq23,rinv23);
898 felec = _mm256_mul_pd(velec,rinvsq23);
900 /* Update potential sum for this i atom from the interaction with this j atom. */
901 velec = _mm256_andnot_pd(dummy_mask,velec);
902 velecsum = _mm256_add_pd(velecsum,velec);
906 fscal = _mm256_andnot_pd(dummy_mask,fscal);
908 /* Calculate temporary vectorial force */
909 tx = _mm256_mul_pd(fscal,dx23);
910 ty = _mm256_mul_pd(fscal,dy23);
911 tz = _mm256_mul_pd(fscal,dz23);
913 /* Update vectorial force */
914 fix2 = _mm256_add_pd(fix2,tx);
915 fiy2 = _mm256_add_pd(fiy2,ty);
916 fiz2 = _mm256_add_pd(fiz2,tz);
918 fjx3 = _mm256_add_pd(fjx3,tx);
919 fjy3 = _mm256_add_pd(fjy3,ty);
920 fjz3 = _mm256_add_pd(fjz3,tz);
922 /**************************
923 * CALCULATE INTERACTIONS *
924 **************************/
926 /* COULOMB ELECTROSTATICS */
927 velec = _mm256_mul_pd(qq31,rinv31);
928 felec = _mm256_mul_pd(velec,rinvsq31);
930 /* Update potential sum for this i atom from the interaction with this j atom. */
931 velec = _mm256_andnot_pd(dummy_mask,velec);
932 velecsum = _mm256_add_pd(velecsum,velec);
936 fscal = _mm256_andnot_pd(dummy_mask,fscal);
938 /* Calculate temporary vectorial force */
939 tx = _mm256_mul_pd(fscal,dx31);
940 ty = _mm256_mul_pd(fscal,dy31);
941 tz = _mm256_mul_pd(fscal,dz31);
943 /* Update vectorial force */
944 fix3 = _mm256_add_pd(fix3,tx);
945 fiy3 = _mm256_add_pd(fiy3,ty);
946 fiz3 = _mm256_add_pd(fiz3,tz);
948 fjx1 = _mm256_add_pd(fjx1,tx);
949 fjy1 = _mm256_add_pd(fjy1,ty);
950 fjz1 = _mm256_add_pd(fjz1,tz);
952 /**************************
953 * CALCULATE INTERACTIONS *
954 **************************/
956 /* COULOMB ELECTROSTATICS */
957 velec = _mm256_mul_pd(qq32,rinv32);
958 felec = _mm256_mul_pd(velec,rinvsq32);
960 /* Update potential sum for this i atom from the interaction with this j atom. */
961 velec = _mm256_andnot_pd(dummy_mask,velec);
962 velecsum = _mm256_add_pd(velecsum,velec);
966 fscal = _mm256_andnot_pd(dummy_mask,fscal);
968 /* Calculate temporary vectorial force */
969 tx = _mm256_mul_pd(fscal,dx32);
970 ty = _mm256_mul_pd(fscal,dy32);
971 tz = _mm256_mul_pd(fscal,dz32);
973 /* Update vectorial force */
974 fix3 = _mm256_add_pd(fix3,tx);
975 fiy3 = _mm256_add_pd(fiy3,ty);
976 fiz3 = _mm256_add_pd(fiz3,tz);
978 fjx2 = _mm256_add_pd(fjx2,tx);
979 fjy2 = _mm256_add_pd(fjy2,ty);
980 fjz2 = _mm256_add_pd(fjz2,tz);
982 /**************************
983 * CALCULATE INTERACTIONS *
984 **************************/
986 /* COULOMB ELECTROSTATICS */
987 velec = _mm256_mul_pd(qq33,rinv33);
988 felec = _mm256_mul_pd(velec,rinvsq33);
990 /* Update potential sum for this i atom from the interaction with this j atom. */
991 velec = _mm256_andnot_pd(dummy_mask,velec);
992 velecsum = _mm256_add_pd(velecsum,velec);
996 fscal = _mm256_andnot_pd(dummy_mask,fscal);
998 /* Calculate temporary vectorial force */
999 tx = _mm256_mul_pd(fscal,dx33);
1000 ty = _mm256_mul_pd(fscal,dy33);
1001 tz = _mm256_mul_pd(fscal,dz33);
1003 /* Update vectorial force */
1004 fix3 = _mm256_add_pd(fix3,tx);
1005 fiy3 = _mm256_add_pd(fiy3,ty);
1006 fiz3 = _mm256_add_pd(fiz3,tz);
1008 fjx3 = _mm256_add_pd(fjx3,tx);
1009 fjy3 = _mm256_add_pd(fjy3,ty);
1010 fjz3 = _mm256_add_pd(fjz3,tz);
1012 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1013 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1014 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1015 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1017 gmx_mm256_decrement_4rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,
1018 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
1019 fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1021 /* Inner loop uses 278 flops */
1024 /* End of innermost loop */
1026 gmx_mm256_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1027 f+i_coord_offset,fshift+i_shift_offset);
1030 /* Update potential energies */
1031 gmx_mm256_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
1032 gmx_mm256_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
1034 /* Increment number of inner iterations */
1035 inneriter += j_index_end - j_index_start;
1037 /* Outer loop uses 26 flops */
1040 /* Increment number of outer iterations */
1043 /* Update outer/inner flops */
1045 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_VF,outeriter*26 + inneriter*278);
1048 * Gromacs nonbonded kernel: nb_kernel_ElecCoul_VdwLJ_GeomW4W4_F_avx_256_double
1049 * Electrostatics interaction: Coulomb
1050 * VdW interaction: LennardJones
1051 * Geometry: Water4-Water4
1052 * Calculate force/pot: Force
1055 nb_kernel_ElecCoul_VdwLJ_GeomW4W4_F_avx_256_double
1056 (t_nblist * gmx_restrict nlist,
1057 rvec * gmx_restrict xx,
1058 rvec * gmx_restrict ff,
1059 t_forcerec * gmx_restrict fr,
1060 t_mdatoms * gmx_restrict mdatoms,
1061 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
1062 t_nrnb * gmx_restrict nrnb)
1064 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
1065 * just 0 for non-waters.
1066 * Suffixes A,B,C,D refer to j loop unrolling done with AVX, e.g. for the four different
1067 * jnr indices corresponding to data put in the four positions in the SIMD register.
1069 int i_shift_offset,i_coord_offset,outeriter,inneriter;
1070 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
1071 int jnrA,jnrB,jnrC,jnrD;
1072 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
1073 int jnrlistE,jnrlistF,jnrlistG,jnrlistH;
1074 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
1075 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
1076 real rcutoff_scalar;
1077 real *shiftvec,*fshift,*x,*f;
1078 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
1079 real scratch[4*DIM];
1080 __m256d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
1081 real * vdwioffsetptr0;
1082 __m256d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
1083 real * vdwioffsetptr1;
1084 __m256d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
1085 real * vdwioffsetptr2;
1086 __m256d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
1087 real * vdwioffsetptr3;
1088 __m256d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
1089 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
1090 __m256d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
1091 int vdwjidx1A,vdwjidx1B,vdwjidx1C,vdwjidx1D;
1092 __m256d jx1,jy1,jz1,fjx1,fjy1,fjz1,jq1,isaj1;
1093 int vdwjidx2A,vdwjidx2B,vdwjidx2C,vdwjidx2D;
1094 __m256d jx2,jy2,jz2,fjx2,fjy2,fjz2,jq2,isaj2;
1095 int vdwjidx3A,vdwjidx3B,vdwjidx3C,vdwjidx3D;
1096 __m256d jx3,jy3,jz3,fjx3,fjy3,fjz3,jq3,isaj3;
1097 __m256d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
1098 __m256d dx11,dy11,dz11,rsq11,rinv11,rinvsq11,r11,qq11,c6_11,c12_11;
1099 __m256d dx12,dy12,dz12,rsq12,rinv12,rinvsq12,r12,qq12,c6_12,c12_12;
1100 __m256d dx13,dy13,dz13,rsq13,rinv13,rinvsq13,r13,qq13,c6_13,c12_13;
1101 __m256d dx21,dy21,dz21,rsq21,rinv21,rinvsq21,r21,qq21,c6_21,c12_21;
1102 __m256d dx22,dy22,dz22,rsq22,rinv22,rinvsq22,r22,qq22,c6_22,c12_22;
1103 __m256d dx23,dy23,dz23,rsq23,rinv23,rinvsq23,r23,qq23,c6_23,c12_23;
1104 __m256d dx31,dy31,dz31,rsq31,rinv31,rinvsq31,r31,qq31,c6_31,c12_31;
1105 __m256d dx32,dy32,dz32,rsq32,rinv32,rinvsq32,r32,qq32,c6_32,c12_32;
1106 __m256d dx33,dy33,dz33,rsq33,rinv33,rinvsq33,r33,qq33,c6_33,c12_33;
1107 __m256d velec,felec,velecsum,facel,crf,krf,krf2;
1110 __m256d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
1113 __m256d one_sixth = _mm256_set1_pd(1.0/6.0);
1114 __m256d one_twelfth = _mm256_set1_pd(1.0/12.0);
1115 __m256d dummy_mask,cutoff_mask;
1116 __m128 tmpmask0,tmpmask1;
1117 __m256d signbit = _mm256_castsi256_pd( _mm256_set1_epi32(0x80000000) );
1118 __m256d one = _mm256_set1_pd(1.0);
1119 __m256d two = _mm256_set1_pd(2.0);
1125 jindex = nlist->jindex;
1127 shiftidx = nlist->shift;
1129 shiftvec = fr->shift_vec[0];
1130 fshift = fr->fshift[0];
1131 facel = _mm256_set1_pd(fr->epsfac);
1132 charge = mdatoms->chargeA;
1133 nvdwtype = fr->ntype;
1134 vdwparam = fr->nbfp;
1135 vdwtype = mdatoms->typeA;
1137 /* Setup water-specific parameters */
1138 inr = nlist->iinr[0];
1139 iq1 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+1]));
1140 iq2 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+2]));
1141 iq3 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+3]));
1142 vdwioffsetptr0 = vdwparam+2*nvdwtype*vdwtype[inr+0];
1144 jq1 = _mm256_set1_pd(charge[inr+1]);
1145 jq2 = _mm256_set1_pd(charge[inr+2]);
1146 jq3 = _mm256_set1_pd(charge[inr+3]);
1147 vdwjidx0A = 2*vdwtype[inr+0];
1148 c6_00 = _mm256_set1_pd(vdwioffsetptr0[vdwjidx0A]);
1149 c12_00 = _mm256_set1_pd(vdwioffsetptr0[vdwjidx0A+1]);
1150 qq11 = _mm256_mul_pd(iq1,jq1);
1151 qq12 = _mm256_mul_pd(iq1,jq2);
1152 qq13 = _mm256_mul_pd(iq1,jq3);
1153 qq21 = _mm256_mul_pd(iq2,jq1);
1154 qq22 = _mm256_mul_pd(iq2,jq2);
1155 qq23 = _mm256_mul_pd(iq2,jq3);
1156 qq31 = _mm256_mul_pd(iq3,jq1);
1157 qq32 = _mm256_mul_pd(iq3,jq2);
1158 qq33 = _mm256_mul_pd(iq3,jq3);
1160 /* Avoid stupid compiler warnings */
1161 jnrA = jnrB = jnrC = jnrD = 0;
1162 j_coord_offsetA = 0;
1163 j_coord_offsetB = 0;
1164 j_coord_offsetC = 0;
1165 j_coord_offsetD = 0;
1170 for(iidx=0;iidx<4*DIM;iidx++)
1172 scratch[iidx] = 0.0;
1175 /* Start outer loop over neighborlists */
1176 for(iidx=0; iidx<nri; iidx++)
1178 /* Load shift vector for this list */
1179 i_shift_offset = DIM*shiftidx[iidx];
1181 /* Load limits for loop over neighbors */
1182 j_index_start = jindex[iidx];
1183 j_index_end = jindex[iidx+1];
1185 /* Get outer coordinate index */
1187 i_coord_offset = DIM*inr;
1189 /* Load i particle coords and add shift vector */
1190 gmx_mm256_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
1191 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
1193 fix0 = _mm256_setzero_pd();
1194 fiy0 = _mm256_setzero_pd();
1195 fiz0 = _mm256_setzero_pd();
1196 fix1 = _mm256_setzero_pd();
1197 fiy1 = _mm256_setzero_pd();
1198 fiz1 = _mm256_setzero_pd();
1199 fix2 = _mm256_setzero_pd();
1200 fiy2 = _mm256_setzero_pd();
1201 fiz2 = _mm256_setzero_pd();
1202 fix3 = _mm256_setzero_pd();
1203 fiy3 = _mm256_setzero_pd();
1204 fiz3 = _mm256_setzero_pd();
1206 /* Start inner kernel loop */
1207 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
1210 /* Get j neighbor index, and coordinate index */
1212 jnrB = jjnr[jidx+1];
1213 jnrC = jjnr[jidx+2];
1214 jnrD = jjnr[jidx+3];
1215 j_coord_offsetA = DIM*jnrA;
1216 j_coord_offsetB = DIM*jnrB;
1217 j_coord_offsetC = DIM*jnrC;
1218 j_coord_offsetD = DIM*jnrD;
1220 /* load j atom coordinates */
1221 gmx_mm256_load_4rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1222 x+j_coord_offsetC,x+j_coord_offsetD,
1223 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1224 &jy2,&jz2,&jx3,&jy3,&jz3);
1226 /* Calculate displacement vector */
1227 dx00 = _mm256_sub_pd(ix0,jx0);
1228 dy00 = _mm256_sub_pd(iy0,jy0);
1229 dz00 = _mm256_sub_pd(iz0,jz0);
1230 dx11 = _mm256_sub_pd(ix1,jx1);
1231 dy11 = _mm256_sub_pd(iy1,jy1);
1232 dz11 = _mm256_sub_pd(iz1,jz1);
1233 dx12 = _mm256_sub_pd(ix1,jx2);
1234 dy12 = _mm256_sub_pd(iy1,jy2);
1235 dz12 = _mm256_sub_pd(iz1,jz2);
1236 dx13 = _mm256_sub_pd(ix1,jx3);
1237 dy13 = _mm256_sub_pd(iy1,jy3);
1238 dz13 = _mm256_sub_pd(iz1,jz3);
1239 dx21 = _mm256_sub_pd(ix2,jx1);
1240 dy21 = _mm256_sub_pd(iy2,jy1);
1241 dz21 = _mm256_sub_pd(iz2,jz1);
1242 dx22 = _mm256_sub_pd(ix2,jx2);
1243 dy22 = _mm256_sub_pd(iy2,jy2);
1244 dz22 = _mm256_sub_pd(iz2,jz2);
1245 dx23 = _mm256_sub_pd(ix2,jx3);
1246 dy23 = _mm256_sub_pd(iy2,jy3);
1247 dz23 = _mm256_sub_pd(iz2,jz3);
1248 dx31 = _mm256_sub_pd(ix3,jx1);
1249 dy31 = _mm256_sub_pd(iy3,jy1);
1250 dz31 = _mm256_sub_pd(iz3,jz1);
1251 dx32 = _mm256_sub_pd(ix3,jx2);
1252 dy32 = _mm256_sub_pd(iy3,jy2);
1253 dz32 = _mm256_sub_pd(iz3,jz2);
1254 dx33 = _mm256_sub_pd(ix3,jx3);
1255 dy33 = _mm256_sub_pd(iy3,jy3);
1256 dz33 = _mm256_sub_pd(iz3,jz3);
1258 /* Calculate squared distance and things based on it */
1259 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
1260 rsq11 = gmx_mm256_calc_rsq_pd(dx11,dy11,dz11);
1261 rsq12 = gmx_mm256_calc_rsq_pd(dx12,dy12,dz12);
1262 rsq13 = gmx_mm256_calc_rsq_pd(dx13,dy13,dz13);
1263 rsq21 = gmx_mm256_calc_rsq_pd(dx21,dy21,dz21);
1264 rsq22 = gmx_mm256_calc_rsq_pd(dx22,dy22,dz22);
1265 rsq23 = gmx_mm256_calc_rsq_pd(dx23,dy23,dz23);
1266 rsq31 = gmx_mm256_calc_rsq_pd(dx31,dy31,dz31);
1267 rsq32 = gmx_mm256_calc_rsq_pd(dx32,dy32,dz32);
1268 rsq33 = gmx_mm256_calc_rsq_pd(dx33,dy33,dz33);
1270 rinv11 = gmx_mm256_invsqrt_pd(rsq11);
1271 rinv12 = gmx_mm256_invsqrt_pd(rsq12);
1272 rinv13 = gmx_mm256_invsqrt_pd(rsq13);
1273 rinv21 = gmx_mm256_invsqrt_pd(rsq21);
1274 rinv22 = gmx_mm256_invsqrt_pd(rsq22);
1275 rinv23 = gmx_mm256_invsqrt_pd(rsq23);
1276 rinv31 = gmx_mm256_invsqrt_pd(rsq31);
1277 rinv32 = gmx_mm256_invsqrt_pd(rsq32);
1278 rinv33 = gmx_mm256_invsqrt_pd(rsq33);
1280 rinvsq00 = gmx_mm256_inv_pd(rsq00);
1281 rinvsq11 = _mm256_mul_pd(rinv11,rinv11);
1282 rinvsq12 = _mm256_mul_pd(rinv12,rinv12);
1283 rinvsq13 = _mm256_mul_pd(rinv13,rinv13);
1284 rinvsq21 = _mm256_mul_pd(rinv21,rinv21);
1285 rinvsq22 = _mm256_mul_pd(rinv22,rinv22);
1286 rinvsq23 = _mm256_mul_pd(rinv23,rinv23);
1287 rinvsq31 = _mm256_mul_pd(rinv31,rinv31);
1288 rinvsq32 = _mm256_mul_pd(rinv32,rinv32);
1289 rinvsq33 = _mm256_mul_pd(rinv33,rinv33);
1291 fjx0 = _mm256_setzero_pd();
1292 fjy0 = _mm256_setzero_pd();
1293 fjz0 = _mm256_setzero_pd();
1294 fjx1 = _mm256_setzero_pd();
1295 fjy1 = _mm256_setzero_pd();
1296 fjz1 = _mm256_setzero_pd();
1297 fjx2 = _mm256_setzero_pd();
1298 fjy2 = _mm256_setzero_pd();
1299 fjz2 = _mm256_setzero_pd();
1300 fjx3 = _mm256_setzero_pd();
1301 fjy3 = _mm256_setzero_pd();
1302 fjz3 = _mm256_setzero_pd();
1304 /**************************
1305 * CALCULATE INTERACTIONS *
1306 **************************/
1308 /* LENNARD-JONES DISPERSION/REPULSION */
1310 rinvsix = _mm256_mul_pd(_mm256_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1311 fvdw = _mm256_mul_pd(_mm256_sub_pd(_mm256_mul_pd(c12_00,rinvsix),c6_00),_mm256_mul_pd(rinvsix,rinvsq00));
1315 /* Calculate temporary vectorial force */
1316 tx = _mm256_mul_pd(fscal,dx00);
1317 ty = _mm256_mul_pd(fscal,dy00);
1318 tz = _mm256_mul_pd(fscal,dz00);
1320 /* Update vectorial force */
1321 fix0 = _mm256_add_pd(fix0,tx);
1322 fiy0 = _mm256_add_pd(fiy0,ty);
1323 fiz0 = _mm256_add_pd(fiz0,tz);
1325 fjx0 = _mm256_add_pd(fjx0,tx);
1326 fjy0 = _mm256_add_pd(fjy0,ty);
1327 fjz0 = _mm256_add_pd(fjz0,tz);
1329 /**************************
1330 * CALCULATE INTERACTIONS *
1331 **************************/
1333 /* COULOMB ELECTROSTATICS */
1334 velec = _mm256_mul_pd(qq11,rinv11);
1335 felec = _mm256_mul_pd(velec,rinvsq11);
1339 /* Calculate temporary vectorial force */
1340 tx = _mm256_mul_pd(fscal,dx11);
1341 ty = _mm256_mul_pd(fscal,dy11);
1342 tz = _mm256_mul_pd(fscal,dz11);
1344 /* Update vectorial force */
1345 fix1 = _mm256_add_pd(fix1,tx);
1346 fiy1 = _mm256_add_pd(fiy1,ty);
1347 fiz1 = _mm256_add_pd(fiz1,tz);
1349 fjx1 = _mm256_add_pd(fjx1,tx);
1350 fjy1 = _mm256_add_pd(fjy1,ty);
1351 fjz1 = _mm256_add_pd(fjz1,tz);
1353 /**************************
1354 * CALCULATE INTERACTIONS *
1355 **************************/
1357 /* COULOMB ELECTROSTATICS */
1358 velec = _mm256_mul_pd(qq12,rinv12);
1359 felec = _mm256_mul_pd(velec,rinvsq12);
1363 /* Calculate temporary vectorial force */
1364 tx = _mm256_mul_pd(fscal,dx12);
1365 ty = _mm256_mul_pd(fscal,dy12);
1366 tz = _mm256_mul_pd(fscal,dz12);
1368 /* Update vectorial force */
1369 fix1 = _mm256_add_pd(fix1,tx);
1370 fiy1 = _mm256_add_pd(fiy1,ty);
1371 fiz1 = _mm256_add_pd(fiz1,tz);
1373 fjx2 = _mm256_add_pd(fjx2,tx);
1374 fjy2 = _mm256_add_pd(fjy2,ty);
1375 fjz2 = _mm256_add_pd(fjz2,tz);
1377 /**************************
1378 * CALCULATE INTERACTIONS *
1379 **************************/
1381 /* COULOMB ELECTROSTATICS */
1382 velec = _mm256_mul_pd(qq13,rinv13);
1383 felec = _mm256_mul_pd(velec,rinvsq13);
1387 /* Calculate temporary vectorial force */
1388 tx = _mm256_mul_pd(fscal,dx13);
1389 ty = _mm256_mul_pd(fscal,dy13);
1390 tz = _mm256_mul_pd(fscal,dz13);
1392 /* Update vectorial force */
1393 fix1 = _mm256_add_pd(fix1,tx);
1394 fiy1 = _mm256_add_pd(fiy1,ty);
1395 fiz1 = _mm256_add_pd(fiz1,tz);
1397 fjx3 = _mm256_add_pd(fjx3,tx);
1398 fjy3 = _mm256_add_pd(fjy3,ty);
1399 fjz3 = _mm256_add_pd(fjz3,tz);
1401 /**************************
1402 * CALCULATE INTERACTIONS *
1403 **************************/
1405 /* COULOMB ELECTROSTATICS */
1406 velec = _mm256_mul_pd(qq21,rinv21);
1407 felec = _mm256_mul_pd(velec,rinvsq21);
1411 /* Calculate temporary vectorial force */
1412 tx = _mm256_mul_pd(fscal,dx21);
1413 ty = _mm256_mul_pd(fscal,dy21);
1414 tz = _mm256_mul_pd(fscal,dz21);
1416 /* Update vectorial force */
1417 fix2 = _mm256_add_pd(fix2,tx);
1418 fiy2 = _mm256_add_pd(fiy2,ty);
1419 fiz2 = _mm256_add_pd(fiz2,tz);
1421 fjx1 = _mm256_add_pd(fjx1,tx);
1422 fjy1 = _mm256_add_pd(fjy1,ty);
1423 fjz1 = _mm256_add_pd(fjz1,tz);
1425 /**************************
1426 * CALCULATE INTERACTIONS *
1427 **************************/
1429 /* COULOMB ELECTROSTATICS */
1430 velec = _mm256_mul_pd(qq22,rinv22);
1431 felec = _mm256_mul_pd(velec,rinvsq22);
1435 /* Calculate temporary vectorial force */
1436 tx = _mm256_mul_pd(fscal,dx22);
1437 ty = _mm256_mul_pd(fscal,dy22);
1438 tz = _mm256_mul_pd(fscal,dz22);
1440 /* Update vectorial force */
1441 fix2 = _mm256_add_pd(fix2,tx);
1442 fiy2 = _mm256_add_pd(fiy2,ty);
1443 fiz2 = _mm256_add_pd(fiz2,tz);
1445 fjx2 = _mm256_add_pd(fjx2,tx);
1446 fjy2 = _mm256_add_pd(fjy2,ty);
1447 fjz2 = _mm256_add_pd(fjz2,tz);
1449 /**************************
1450 * CALCULATE INTERACTIONS *
1451 **************************/
1453 /* COULOMB ELECTROSTATICS */
1454 velec = _mm256_mul_pd(qq23,rinv23);
1455 felec = _mm256_mul_pd(velec,rinvsq23);
1459 /* Calculate temporary vectorial force */
1460 tx = _mm256_mul_pd(fscal,dx23);
1461 ty = _mm256_mul_pd(fscal,dy23);
1462 tz = _mm256_mul_pd(fscal,dz23);
1464 /* Update vectorial force */
1465 fix2 = _mm256_add_pd(fix2,tx);
1466 fiy2 = _mm256_add_pd(fiy2,ty);
1467 fiz2 = _mm256_add_pd(fiz2,tz);
1469 fjx3 = _mm256_add_pd(fjx3,tx);
1470 fjy3 = _mm256_add_pd(fjy3,ty);
1471 fjz3 = _mm256_add_pd(fjz3,tz);
1473 /**************************
1474 * CALCULATE INTERACTIONS *
1475 **************************/
1477 /* COULOMB ELECTROSTATICS */
1478 velec = _mm256_mul_pd(qq31,rinv31);
1479 felec = _mm256_mul_pd(velec,rinvsq31);
1483 /* Calculate temporary vectorial force */
1484 tx = _mm256_mul_pd(fscal,dx31);
1485 ty = _mm256_mul_pd(fscal,dy31);
1486 tz = _mm256_mul_pd(fscal,dz31);
1488 /* Update vectorial force */
1489 fix3 = _mm256_add_pd(fix3,tx);
1490 fiy3 = _mm256_add_pd(fiy3,ty);
1491 fiz3 = _mm256_add_pd(fiz3,tz);
1493 fjx1 = _mm256_add_pd(fjx1,tx);
1494 fjy1 = _mm256_add_pd(fjy1,ty);
1495 fjz1 = _mm256_add_pd(fjz1,tz);
1497 /**************************
1498 * CALCULATE INTERACTIONS *
1499 **************************/
1501 /* COULOMB ELECTROSTATICS */
1502 velec = _mm256_mul_pd(qq32,rinv32);
1503 felec = _mm256_mul_pd(velec,rinvsq32);
1507 /* Calculate temporary vectorial force */
1508 tx = _mm256_mul_pd(fscal,dx32);
1509 ty = _mm256_mul_pd(fscal,dy32);
1510 tz = _mm256_mul_pd(fscal,dz32);
1512 /* Update vectorial force */
1513 fix3 = _mm256_add_pd(fix3,tx);
1514 fiy3 = _mm256_add_pd(fiy3,ty);
1515 fiz3 = _mm256_add_pd(fiz3,tz);
1517 fjx2 = _mm256_add_pd(fjx2,tx);
1518 fjy2 = _mm256_add_pd(fjy2,ty);
1519 fjz2 = _mm256_add_pd(fjz2,tz);
1521 /**************************
1522 * CALCULATE INTERACTIONS *
1523 **************************/
1525 /* COULOMB ELECTROSTATICS */
1526 velec = _mm256_mul_pd(qq33,rinv33);
1527 felec = _mm256_mul_pd(velec,rinvsq33);
1531 /* Calculate temporary vectorial force */
1532 tx = _mm256_mul_pd(fscal,dx33);
1533 ty = _mm256_mul_pd(fscal,dy33);
1534 tz = _mm256_mul_pd(fscal,dz33);
1536 /* Update vectorial force */
1537 fix3 = _mm256_add_pd(fix3,tx);
1538 fiy3 = _mm256_add_pd(fiy3,ty);
1539 fiz3 = _mm256_add_pd(fiz3,tz);
1541 fjx3 = _mm256_add_pd(fjx3,tx);
1542 fjy3 = _mm256_add_pd(fjy3,ty);
1543 fjz3 = _mm256_add_pd(fjz3,tz);
1545 fjptrA = f+j_coord_offsetA;
1546 fjptrB = f+j_coord_offsetB;
1547 fjptrC = f+j_coord_offsetC;
1548 fjptrD = f+j_coord_offsetD;
1550 gmx_mm256_decrement_4rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,
1551 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
1552 fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1554 /* Inner loop uses 264 flops */
1557 if(jidx<j_index_end)
1560 /* Get j neighbor index, and coordinate index */
1561 jnrlistA = jjnr[jidx];
1562 jnrlistB = jjnr[jidx+1];
1563 jnrlistC = jjnr[jidx+2];
1564 jnrlistD = jjnr[jidx+3];
1565 /* Sign of each element will be negative for non-real atoms.
1566 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1567 * so use it as val = _mm_andnot_pd(mask,val) to clear dummy entries.
1569 tmpmask0 = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1571 tmpmask1 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(3,3,2,2));
1572 tmpmask0 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(1,1,0,0));
1573 dummy_mask = _mm256_castps_pd(gmx_mm256_set_m128(tmpmask1,tmpmask0));
1575 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1576 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1577 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1578 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1579 j_coord_offsetA = DIM*jnrA;
1580 j_coord_offsetB = DIM*jnrB;
1581 j_coord_offsetC = DIM*jnrC;
1582 j_coord_offsetD = DIM*jnrD;
1584 /* load j atom coordinates */
1585 gmx_mm256_load_4rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1586 x+j_coord_offsetC,x+j_coord_offsetD,
1587 &jx0,&jy0,&jz0,&jx1,&jy1,&jz1,&jx2,
1588 &jy2,&jz2,&jx3,&jy3,&jz3);
1590 /* Calculate displacement vector */
1591 dx00 = _mm256_sub_pd(ix0,jx0);
1592 dy00 = _mm256_sub_pd(iy0,jy0);
1593 dz00 = _mm256_sub_pd(iz0,jz0);
1594 dx11 = _mm256_sub_pd(ix1,jx1);
1595 dy11 = _mm256_sub_pd(iy1,jy1);
1596 dz11 = _mm256_sub_pd(iz1,jz1);
1597 dx12 = _mm256_sub_pd(ix1,jx2);
1598 dy12 = _mm256_sub_pd(iy1,jy2);
1599 dz12 = _mm256_sub_pd(iz1,jz2);
1600 dx13 = _mm256_sub_pd(ix1,jx3);
1601 dy13 = _mm256_sub_pd(iy1,jy3);
1602 dz13 = _mm256_sub_pd(iz1,jz3);
1603 dx21 = _mm256_sub_pd(ix2,jx1);
1604 dy21 = _mm256_sub_pd(iy2,jy1);
1605 dz21 = _mm256_sub_pd(iz2,jz1);
1606 dx22 = _mm256_sub_pd(ix2,jx2);
1607 dy22 = _mm256_sub_pd(iy2,jy2);
1608 dz22 = _mm256_sub_pd(iz2,jz2);
1609 dx23 = _mm256_sub_pd(ix2,jx3);
1610 dy23 = _mm256_sub_pd(iy2,jy3);
1611 dz23 = _mm256_sub_pd(iz2,jz3);
1612 dx31 = _mm256_sub_pd(ix3,jx1);
1613 dy31 = _mm256_sub_pd(iy3,jy1);
1614 dz31 = _mm256_sub_pd(iz3,jz1);
1615 dx32 = _mm256_sub_pd(ix3,jx2);
1616 dy32 = _mm256_sub_pd(iy3,jy2);
1617 dz32 = _mm256_sub_pd(iz3,jz2);
1618 dx33 = _mm256_sub_pd(ix3,jx3);
1619 dy33 = _mm256_sub_pd(iy3,jy3);
1620 dz33 = _mm256_sub_pd(iz3,jz3);
1622 /* Calculate squared distance and things based on it */
1623 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
1624 rsq11 = gmx_mm256_calc_rsq_pd(dx11,dy11,dz11);
1625 rsq12 = gmx_mm256_calc_rsq_pd(dx12,dy12,dz12);
1626 rsq13 = gmx_mm256_calc_rsq_pd(dx13,dy13,dz13);
1627 rsq21 = gmx_mm256_calc_rsq_pd(dx21,dy21,dz21);
1628 rsq22 = gmx_mm256_calc_rsq_pd(dx22,dy22,dz22);
1629 rsq23 = gmx_mm256_calc_rsq_pd(dx23,dy23,dz23);
1630 rsq31 = gmx_mm256_calc_rsq_pd(dx31,dy31,dz31);
1631 rsq32 = gmx_mm256_calc_rsq_pd(dx32,dy32,dz32);
1632 rsq33 = gmx_mm256_calc_rsq_pd(dx33,dy33,dz33);
1634 rinv11 = gmx_mm256_invsqrt_pd(rsq11);
1635 rinv12 = gmx_mm256_invsqrt_pd(rsq12);
1636 rinv13 = gmx_mm256_invsqrt_pd(rsq13);
1637 rinv21 = gmx_mm256_invsqrt_pd(rsq21);
1638 rinv22 = gmx_mm256_invsqrt_pd(rsq22);
1639 rinv23 = gmx_mm256_invsqrt_pd(rsq23);
1640 rinv31 = gmx_mm256_invsqrt_pd(rsq31);
1641 rinv32 = gmx_mm256_invsqrt_pd(rsq32);
1642 rinv33 = gmx_mm256_invsqrt_pd(rsq33);
1644 rinvsq00 = gmx_mm256_inv_pd(rsq00);
1645 rinvsq11 = _mm256_mul_pd(rinv11,rinv11);
1646 rinvsq12 = _mm256_mul_pd(rinv12,rinv12);
1647 rinvsq13 = _mm256_mul_pd(rinv13,rinv13);
1648 rinvsq21 = _mm256_mul_pd(rinv21,rinv21);
1649 rinvsq22 = _mm256_mul_pd(rinv22,rinv22);
1650 rinvsq23 = _mm256_mul_pd(rinv23,rinv23);
1651 rinvsq31 = _mm256_mul_pd(rinv31,rinv31);
1652 rinvsq32 = _mm256_mul_pd(rinv32,rinv32);
1653 rinvsq33 = _mm256_mul_pd(rinv33,rinv33);
1655 fjx0 = _mm256_setzero_pd();
1656 fjy0 = _mm256_setzero_pd();
1657 fjz0 = _mm256_setzero_pd();
1658 fjx1 = _mm256_setzero_pd();
1659 fjy1 = _mm256_setzero_pd();
1660 fjz1 = _mm256_setzero_pd();
1661 fjx2 = _mm256_setzero_pd();
1662 fjy2 = _mm256_setzero_pd();
1663 fjz2 = _mm256_setzero_pd();
1664 fjx3 = _mm256_setzero_pd();
1665 fjy3 = _mm256_setzero_pd();
1666 fjz3 = _mm256_setzero_pd();
1668 /**************************
1669 * CALCULATE INTERACTIONS *
1670 **************************/
1672 /* LENNARD-JONES DISPERSION/REPULSION */
1674 rinvsix = _mm256_mul_pd(_mm256_mul_pd(rinvsq00,rinvsq00),rinvsq00);
1675 fvdw = _mm256_mul_pd(_mm256_sub_pd(_mm256_mul_pd(c12_00,rinvsix),c6_00),_mm256_mul_pd(rinvsix,rinvsq00));
1679 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1681 /* Calculate temporary vectorial force */
1682 tx = _mm256_mul_pd(fscal,dx00);
1683 ty = _mm256_mul_pd(fscal,dy00);
1684 tz = _mm256_mul_pd(fscal,dz00);
1686 /* Update vectorial force */
1687 fix0 = _mm256_add_pd(fix0,tx);
1688 fiy0 = _mm256_add_pd(fiy0,ty);
1689 fiz0 = _mm256_add_pd(fiz0,tz);
1691 fjx0 = _mm256_add_pd(fjx0,tx);
1692 fjy0 = _mm256_add_pd(fjy0,ty);
1693 fjz0 = _mm256_add_pd(fjz0,tz);
1695 /**************************
1696 * CALCULATE INTERACTIONS *
1697 **************************/
1699 /* COULOMB ELECTROSTATICS */
1700 velec = _mm256_mul_pd(qq11,rinv11);
1701 felec = _mm256_mul_pd(velec,rinvsq11);
1705 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1707 /* Calculate temporary vectorial force */
1708 tx = _mm256_mul_pd(fscal,dx11);
1709 ty = _mm256_mul_pd(fscal,dy11);
1710 tz = _mm256_mul_pd(fscal,dz11);
1712 /* Update vectorial force */
1713 fix1 = _mm256_add_pd(fix1,tx);
1714 fiy1 = _mm256_add_pd(fiy1,ty);
1715 fiz1 = _mm256_add_pd(fiz1,tz);
1717 fjx1 = _mm256_add_pd(fjx1,tx);
1718 fjy1 = _mm256_add_pd(fjy1,ty);
1719 fjz1 = _mm256_add_pd(fjz1,tz);
1721 /**************************
1722 * CALCULATE INTERACTIONS *
1723 **************************/
1725 /* COULOMB ELECTROSTATICS */
1726 velec = _mm256_mul_pd(qq12,rinv12);
1727 felec = _mm256_mul_pd(velec,rinvsq12);
1731 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1733 /* Calculate temporary vectorial force */
1734 tx = _mm256_mul_pd(fscal,dx12);
1735 ty = _mm256_mul_pd(fscal,dy12);
1736 tz = _mm256_mul_pd(fscal,dz12);
1738 /* Update vectorial force */
1739 fix1 = _mm256_add_pd(fix1,tx);
1740 fiy1 = _mm256_add_pd(fiy1,ty);
1741 fiz1 = _mm256_add_pd(fiz1,tz);
1743 fjx2 = _mm256_add_pd(fjx2,tx);
1744 fjy2 = _mm256_add_pd(fjy2,ty);
1745 fjz2 = _mm256_add_pd(fjz2,tz);
1747 /**************************
1748 * CALCULATE INTERACTIONS *
1749 **************************/
1751 /* COULOMB ELECTROSTATICS */
1752 velec = _mm256_mul_pd(qq13,rinv13);
1753 felec = _mm256_mul_pd(velec,rinvsq13);
1757 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1759 /* Calculate temporary vectorial force */
1760 tx = _mm256_mul_pd(fscal,dx13);
1761 ty = _mm256_mul_pd(fscal,dy13);
1762 tz = _mm256_mul_pd(fscal,dz13);
1764 /* Update vectorial force */
1765 fix1 = _mm256_add_pd(fix1,tx);
1766 fiy1 = _mm256_add_pd(fiy1,ty);
1767 fiz1 = _mm256_add_pd(fiz1,tz);
1769 fjx3 = _mm256_add_pd(fjx3,tx);
1770 fjy3 = _mm256_add_pd(fjy3,ty);
1771 fjz3 = _mm256_add_pd(fjz3,tz);
1773 /**************************
1774 * CALCULATE INTERACTIONS *
1775 **************************/
1777 /* COULOMB ELECTROSTATICS */
1778 velec = _mm256_mul_pd(qq21,rinv21);
1779 felec = _mm256_mul_pd(velec,rinvsq21);
1783 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1785 /* Calculate temporary vectorial force */
1786 tx = _mm256_mul_pd(fscal,dx21);
1787 ty = _mm256_mul_pd(fscal,dy21);
1788 tz = _mm256_mul_pd(fscal,dz21);
1790 /* Update vectorial force */
1791 fix2 = _mm256_add_pd(fix2,tx);
1792 fiy2 = _mm256_add_pd(fiy2,ty);
1793 fiz2 = _mm256_add_pd(fiz2,tz);
1795 fjx1 = _mm256_add_pd(fjx1,tx);
1796 fjy1 = _mm256_add_pd(fjy1,ty);
1797 fjz1 = _mm256_add_pd(fjz1,tz);
1799 /**************************
1800 * CALCULATE INTERACTIONS *
1801 **************************/
1803 /* COULOMB ELECTROSTATICS */
1804 velec = _mm256_mul_pd(qq22,rinv22);
1805 felec = _mm256_mul_pd(velec,rinvsq22);
1809 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1811 /* Calculate temporary vectorial force */
1812 tx = _mm256_mul_pd(fscal,dx22);
1813 ty = _mm256_mul_pd(fscal,dy22);
1814 tz = _mm256_mul_pd(fscal,dz22);
1816 /* Update vectorial force */
1817 fix2 = _mm256_add_pd(fix2,tx);
1818 fiy2 = _mm256_add_pd(fiy2,ty);
1819 fiz2 = _mm256_add_pd(fiz2,tz);
1821 fjx2 = _mm256_add_pd(fjx2,tx);
1822 fjy2 = _mm256_add_pd(fjy2,ty);
1823 fjz2 = _mm256_add_pd(fjz2,tz);
1825 /**************************
1826 * CALCULATE INTERACTIONS *
1827 **************************/
1829 /* COULOMB ELECTROSTATICS */
1830 velec = _mm256_mul_pd(qq23,rinv23);
1831 felec = _mm256_mul_pd(velec,rinvsq23);
1835 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1837 /* Calculate temporary vectorial force */
1838 tx = _mm256_mul_pd(fscal,dx23);
1839 ty = _mm256_mul_pd(fscal,dy23);
1840 tz = _mm256_mul_pd(fscal,dz23);
1842 /* Update vectorial force */
1843 fix2 = _mm256_add_pd(fix2,tx);
1844 fiy2 = _mm256_add_pd(fiy2,ty);
1845 fiz2 = _mm256_add_pd(fiz2,tz);
1847 fjx3 = _mm256_add_pd(fjx3,tx);
1848 fjy3 = _mm256_add_pd(fjy3,ty);
1849 fjz3 = _mm256_add_pd(fjz3,tz);
1851 /**************************
1852 * CALCULATE INTERACTIONS *
1853 **************************/
1855 /* COULOMB ELECTROSTATICS */
1856 velec = _mm256_mul_pd(qq31,rinv31);
1857 felec = _mm256_mul_pd(velec,rinvsq31);
1861 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1863 /* Calculate temporary vectorial force */
1864 tx = _mm256_mul_pd(fscal,dx31);
1865 ty = _mm256_mul_pd(fscal,dy31);
1866 tz = _mm256_mul_pd(fscal,dz31);
1868 /* Update vectorial force */
1869 fix3 = _mm256_add_pd(fix3,tx);
1870 fiy3 = _mm256_add_pd(fiy3,ty);
1871 fiz3 = _mm256_add_pd(fiz3,tz);
1873 fjx1 = _mm256_add_pd(fjx1,tx);
1874 fjy1 = _mm256_add_pd(fjy1,ty);
1875 fjz1 = _mm256_add_pd(fjz1,tz);
1877 /**************************
1878 * CALCULATE INTERACTIONS *
1879 **************************/
1881 /* COULOMB ELECTROSTATICS */
1882 velec = _mm256_mul_pd(qq32,rinv32);
1883 felec = _mm256_mul_pd(velec,rinvsq32);
1887 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1889 /* Calculate temporary vectorial force */
1890 tx = _mm256_mul_pd(fscal,dx32);
1891 ty = _mm256_mul_pd(fscal,dy32);
1892 tz = _mm256_mul_pd(fscal,dz32);
1894 /* Update vectorial force */
1895 fix3 = _mm256_add_pd(fix3,tx);
1896 fiy3 = _mm256_add_pd(fiy3,ty);
1897 fiz3 = _mm256_add_pd(fiz3,tz);
1899 fjx2 = _mm256_add_pd(fjx2,tx);
1900 fjy2 = _mm256_add_pd(fjy2,ty);
1901 fjz2 = _mm256_add_pd(fjz2,tz);
1903 /**************************
1904 * CALCULATE INTERACTIONS *
1905 **************************/
1907 /* COULOMB ELECTROSTATICS */
1908 velec = _mm256_mul_pd(qq33,rinv33);
1909 felec = _mm256_mul_pd(velec,rinvsq33);
1913 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1915 /* Calculate temporary vectorial force */
1916 tx = _mm256_mul_pd(fscal,dx33);
1917 ty = _mm256_mul_pd(fscal,dy33);
1918 tz = _mm256_mul_pd(fscal,dz33);
1920 /* Update vectorial force */
1921 fix3 = _mm256_add_pd(fix3,tx);
1922 fiy3 = _mm256_add_pd(fiy3,ty);
1923 fiz3 = _mm256_add_pd(fiz3,tz);
1925 fjx3 = _mm256_add_pd(fjx3,tx);
1926 fjy3 = _mm256_add_pd(fjy3,ty);
1927 fjz3 = _mm256_add_pd(fjz3,tz);
1929 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1930 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1931 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1932 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1934 gmx_mm256_decrement_4rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,
1935 fjx0,fjy0,fjz0,fjx1,fjy1,fjz1,
1936 fjx2,fjy2,fjz2,fjx3,fjy3,fjz3);
1938 /* Inner loop uses 264 flops */
1941 /* End of innermost loop */
1943 gmx_mm256_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1944 f+i_coord_offset,fshift+i_shift_offset);
1946 /* Increment number of inner iterations */
1947 inneriter += j_index_end - j_index_start;
1949 /* Outer loop uses 24 flops */
1952 /* Increment number of outer iterations */
1955 /* Update outer/inner flops */
1957 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4W4_F,outeriter*24 + inneriter*264);