2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
36 * Note: this file was generated by the GROMACS avx_256_double kernel generator.
44 #include "../nb_kernel.h"
45 #include "gromacs/legacyheaders/types/simple.h"
46 #include "gromacs/math/vec.h"
47 #include "gromacs/legacyheaders/nrnb.h"
49 #include "gromacs/simd/math_x86_avx_256_double.h"
50 #include "kernelutil_x86_avx_256_double.h"
53 * Gromacs nonbonded kernel: nb_kernel_ElecRF_VdwCSTab_GeomW4P1_VF_avx_256_double
54 * Electrostatics interaction: ReactionField
55 * VdW interaction: CubicSplineTable
56 * Geometry: Water4-Particle
57 * Calculate force/pot: PotentialAndForce
60 nb_kernel_ElecRF_VdwCSTab_GeomW4P1_VF_avx_256_double
61 (t_nblist * gmx_restrict nlist,
62 rvec * gmx_restrict xx,
63 rvec * gmx_restrict ff,
64 t_forcerec * gmx_restrict fr,
65 t_mdatoms * gmx_restrict mdatoms,
66 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
67 t_nrnb * gmx_restrict nrnb)
69 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
70 * just 0 for non-waters.
71 * Suffixes A,B,C,D refer to j loop unrolling done with AVX, e.g. for the four different
72 * jnr indices corresponding to data put in the four positions in the SIMD register.
74 int i_shift_offset,i_coord_offset,outeriter,inneriter;
75 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
76 int jnrA,jnrB,jnrC,jnrD;
77 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
78 int jnrlistE,jnrlistF,jnrlistG,jnrlistH;
79 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
80 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
82 real *shiftvec,*fshift,*x,*f;
83 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
85 __m256d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
86 real * vdwioffsetptr0;
87 __m256d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
88 real * vdwioffsetptr1;
89 __m256d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
90 real * vdwioffsetptr2;
91 __m256d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
92 real * vdwioffsetptr3;
93 __m256d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
94 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
95 __m256d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
96 __m256d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
97 __m256d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
98 __m256d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
99 __m256d dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
100 __m256d velec,felec,velecsum,facel,crf,krf,krf2;
103 __m256d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
106 __m256d one_sixth = _mm256_set1_pd(1.0/6.0);
107 __m256d one_twelfth = _mm256_set1_pd(1.0/12.0);
109 __m128i ifour = _mm_set1_epi32(4);
110 __m256d rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
112 __m256d dummy_mask,cutoff_mask;
113 __m128 tmpmask0,tmpmask1;
114 __m256d signbit = _mm256_castsi256_pd( _mm256_set1_epi32(0x80000000) );
115 __m256d one = _mm256_set1_pd(1.0);
116 __m256d two = _mm256_set1_pd(2.0);
122 jindex = nlist->jindex;
124 shiftidx = nlist->shift;
126 shiftvec = fr->shift_vec[0];
127 fshift = fr->fshift[0];
128 facel = _mm256_set1_pd(fr->epsfac);
129 charge = mdatoms->chargeA;
130 krf = _mm256_set1_pd(fr->ic->k_rf);
131 krf2 = _mm256_set1_pd(fr->ic->k_rf*2.0);
132 crf = _mm256_set1_pd(fr->ic->c_rf);
133 nvdwtype = fr->ntype;
135 vdwtype = mdatoms->typeA;
137 vftab = kernel_data->table_vdw->data;
138 vftabscale = _mm256_set1_pd(kernel_data->table_vdw->scale);
140 /* Setup water-specific parameters */
141 inr = nlist->iinr[0];
142 iq1 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+1]));
143 iq2 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+2]));
144 iq3 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+3]));
145 vdwioffsetptr0 = vdwparam+2*nvdwtype*vdwtype[inr+0];
147 /* Avoid stupid compiler warnings */
148 jnrA = jnrB = jnrC = jnrD = 0;
157 for(iidx=0;iidx<4*DIM;iidx++)
162 /* Start outer loop over neighborlists */
163 for(iidx=0; iidx<nri; iidx++)
165 /* Load shift vector for this list */
166 i_shift_offset = DIM*shiftidx[iidx];
168 /* Load limits for loop over neighbors */
169 j_index_start = jindex[iidx];
170 j_index_end = jindex[iidx+1];
172 /* Get outer coordinate index */
174 i_coord_offset = DIM*inr;
176 /* Load i particle coords and add shift vector */
177 gmx_mm256_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
178 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
180 fix0 = _mm256_setzero_pd();
181 fiy0 = _mm256_setzero_pd();
182 fiz0 = _mm256_setzero_pd();
183 fix1 = _mm256_setzero_pd();
184 fiy1 = _mm256_setzero_pd();
185 fiz1 = _mm256_setzero_pd();
186 fix2 = _mm256_setzero_pd();
187 fiy2 = _mm256_setzero_pd();
188 fiz2 = _mm256_setzero_pd();
189 fix3 = _mm256_setzero_pd();
190 fiy3 = _mm256_setzero_pd();
191 fiz3 = _mm256_setzero_pd();
193 /* Reset potential sums */
194 velecsum = _mm256_setzero_pd();
195 vvdwsum = _mm256_setzero_pd();
197 /* Start inner kernel loop */
198 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
201 /* Get j neighbor index, and coordinate index */
206 j_coord_offsetA = DIM*jnrA;
207 j_coord_offsetB = DIM*jnrB;
208 j_coord_offsetC = DIM*jnrC;
209 j_coord_offsetD = DIM*jnrD;
211 /* load j atom coordinates */
212 gmx_mm256_load_1rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
213 x+j_coord_offsetC,x+j_coord_offsetD,
216 /* Calculate displacement vector */
217 dx00 = _mm256_sub_pd(ix0,jx0);
218 dy00 = _mm256_sub_pd(iy0,jy0);
219 dz00 = _mm256_sub_pd(iz0,jz0);
220 dx10 = _mm256_sub_pd(ix1,jx0);
221 dy10 = _mm256_sub_pd(iy1,jy0);
222 dz10 = _mm256_sub_pd(iz1,jz0);
223 dx20 = _mm256_sub_pd(ix2,jx0);
224 dy20 = _mm256_sub_pd(iy2,jy0);
225 dz20 = _mm256_sub_pd(iz2,jz0);
226 dx30 = _mm256_sub_pd(ix3,jx0);
227 dy30 = _mm256_sub_pd(iy3,jy0);
228 dz30 = _mm256_sub_pd(iz3,jz0);
230 /* Calculate squared distance and things based on it */
231 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
232 rsq10 = gmx_mm256_calc_rsq_pd(dx10,dy10,dz10);
233 rsq20 = gmx_mm256_calc_rsq_pd(dx20,dy20,dz20);
234 rsq30 = gmx_mm256_calc_rsq_pd(dx30,dy30,dz30);
236 rinv00 = gmx_mm256_invsqrt_pd(rsq00);
237 rinv10 = gmx_mm256_invsqrt_pd(rsq10);
238 rinv20 = gmx_mm256_invsqrt_pd(rsq20);
239 rinv30 = gmx_mm256_invsqrt_pd(rsq30);
241 rinvsq10 = _mm256_mul_pd(rinv10,rinv10);
242 rinvsq20 = _mm256_mul_pd(rinv20,rinv20);
243 rinvsq30 = _mm256_mul_pd(rinv30,rinv30);
245 /* Load parameters for j particles */
246 jq0 = gmx_mm256_load_4real_swizzle_pd(charge+jnrA+0,charge+jnrB+0,
247 charge+jnrC+0,charge+jnrD+0);
248 vdwjidx0A = 2*vdwtype[jnrA+0];
249 vdwjidx0B = 2*vdwtype[jnrB+0];
250 vdwjidx0C = 2*vdwtype[jnrC+0];
251 vdwjidx0D = 2*vdwtype[jnrD+0];
253 fjx0 = _mm256_setzero_pd();
254 fjy0 = _mm256_setzero_pd();
255 fjz0 = _mm256_setzero_pd();
257 /**************************
258 * CALCULATE INTERACTIONS *
259 **************************/
261 r00 = _mm256_mul_pd(rsq00,rinv00);
263 /* Compute parameters for interactions between i and j atoms */
264 gmx_mm256_load_4pair_swizzle_pd(vdwioffsetptr0+vdwjidx0A,
265 vdwioffsetptr0+vdwjidx0B,
266 vdwioffsetptr0+vdwjidx0C,
267 vdwioffsetptr0+vdwjidx0D,
270 /* Calculate table index by multiplying r with table scale and truncate to integer */
271 rt = _mm256_mul_pd(r00,vftabscale);
272 vfitab = _mm256_cvttpd_epi32(rt);
273 vfeps = _mm256_sub_pd(rt,_mm256_round_pd(rt, _MM_FROUND_FLOOR));
274 vfitab = _mm_slli_epi32(vfitab,3);
276 /* CUBIC SPLINE TABLE DISPERSION */
277 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
278 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
279 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
280 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
281 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
282 Heps = _mm256_mul_pd(vfeps,H);
283 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
284 VV = _mm256_add_pd(Y,_mm256_mul_pd(vfeps,Fp));
285 vvdw6 = _mm256_mul_pd(c6_00,VV);
286 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
287 fvdw6 = _mm256_mul_pd(c6_00,FF);
289 /* CUBIC SPLINE TABLE REPULSION */
290 vfitab = _mm_add_epi32(vfitab,ifour);
291 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
292 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
293 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
294 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
295 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
296 Heps = _mm256_mul_pd(vfeps,H);
297 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
298 VV = _mm256_add_pd(Y,_mm256_mul_pd(vfeps,Fp));
299 vvdw12 = _mm256_mul_pd(c12_00,VV);
300 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
301 fvdw12 = _mm256_mul_pd(c12_00,FF);
302 vvdw = _mm256_add_pd(vvdw12,vvdw6);
303 fvdw = _mm256_xor_pd(signbit,_mm256_mul_pd(_mm256_add_pd(fvdw6,fvdw12),_mm256_mul_pd(vftabscale,rinv00)));
305 /* Update potential sum for this i atom from the interaction with this j atom. */
306 vvdwsum = _mm256_add_pd(vvdwsum,vvdw);
310 /* Calculate temporary vectorial force */
311 tx = _mm256_mul_pd(fscal,dx00);
312 ty = _mm256_mul_pd(fscal,dy00);
313 tz = _mm256_mul_pd(fscal,dz00);
315 /* Update vectorial force */
316 fix0 = _mm256_add_pd(fix0,tx);
317 fiy0 = _mm256_add_pd(fiy0,ty);
318 fiz0 = _mm256_add_pd(fiz0,tz);
320 fjx0 = _mm256_add_pd(fjx0,tx);
321 fjy0 = _mm256_add_pd(fjy0,ty);
322 fjz0 = _mm256_add_pd(fjz0,tz);
324 /**************************
325 * CALCULATE INTERACTIONS *
326 **************************/
328 /* Compute parameters for interactions between i and j atoms */
329 qq10 = _mm256_mul_pd(iq1,jq0);
331 /* REACTION-FIELD ELECTROSTATICS */
332 velec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_add_pd(rinv10,_mm256_mul_pd(krf,rsq10)),crf));
333 felec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_mul_pd(rinv10,rinvsq10),krf2));
335 /* Update potential sum for this i atom from the interaction with this j atom. */
336 velecsum = _mm256_add_pd(velecsum,velec);
340 /* Calculate temporary vectorial force */
341 tx = _mm256_mul_pd(fscal,dx10);
342 ty = _mm256_mul_pd(fscal,dy10);
343 tz = _mm256_mul_pd(fscal,dz10);
345 /* Update vectorial force */
346 fix1 = _mm256_add_pd(fix1,tx);
347 fiy1 = _mm256_add_pd(fiy1,ty);
348 fiz1 = _mm256_add_pd(fiz1,tz);
350 fjx0 = _mm256_add_pd(fjx0,tx);
351 fjy0 = _mm256_add_pd(fjy0,ty);
352 fjz0 = _mm256_add_pd(fjz0,tz);
354 /**************************
355 * CALCULATE INTERACTIONS *
356 **************************/
358 /* Compute parameters for interactions between i and j atoms */
359 qq20 = _mm256_mul_pd(iq2,jq0);
361 /* REACTION-FIELD ELECTROSTATICS */
362 velec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_add_pd(rinv20,_mm256_mul_pd(krf,rsq20)),crf));
363 felec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_mul_pd(rinv20,rinvsq20),krf2));
365 /* Update potential sum for this i atom from the interaction with this j atom. */
366 velecsum = _mm256_add_pd(velecsum,velec);
370 /* Calculate temporary vectorial force */
371 tx = _mm256_mul_pd(fscal,dx20);
372 ty = _mm256_mul_pd(fscal,dy20);
373 tz = _mm256_mul_pd(fscal,dz20);
375 /* Update vectorial force */
376 fix2 = _mm256_add_pd(fix2,tx);
377 fiy2 = _mm256_add_pd(fiy2,ty);
378 fiz2 = _mm256_add_pd(fiz2,tz);
380 fjx0 = _mm256_add_pd(fjx0,tx);
381 fjy0 = _mm256_add_pd(fjy0,ty);
382 fjz0 = _mm256_add_pd(fjz0,tz);
384 /**************************
385 * CALCULATE INTERACTIONS *
386 **************************/
388 /* Compute parameters for interactions between i and j atoms */
389 qq30 = _mm256_mul_pd(iq3,jq0);
391 /* REACTION-FIELD ELECTROSTATICS */
392 velec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_add_pd(rinv30,_mm256_mul_pd(krf,rsq30)),crf));
393 felec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_mul_pd(rinv30,rinvsq30),krf2));
395 /* Update potential sum for this i atom from the interaction with this j atom. */
396 velecsum = _mm256_add_pd(velecsum,velec);
400 /* Calculate temporary vectorial force */
401 tx = _mm256_mul_pd(fscal,dx30);
402 ty = _mm256_mul_pd(fscal,dy30);
403 tz = _mm256_mul_pd(fscal,dz30);
405 /* Update vectorial force */
406 fix3 = _mm256_add_pd(fix3,tx);
407 fiy3 = _mm256_add_pd(fiy3,ty);
408 fiz3 = _mm256_add_pd(fiz3,tz);
410 fjx0 = _mm256_add_pd(fjx0,tx);
411 fjy0 = _mm256_add_pd(fjy0,ty);
412 fjz0 = _mm256_add_pd(fjz0,tz);
414 fjptrA = f+j_coord_offsetA;
415 fjptrB = f+j_coord_offsetB;
416 fjptrC = f+j_coord_offsetC;
417 fjptrD = f+j_coord_offsetD;
419 gmx_mm256_decrement_1rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
421 /* Inner loop uses 155 flops */
427 /* Get j neighbor index, and coordinate index */
428 jnrlistA = jjnr[jidx];
429 jnrlistB = jjnr[jidx+1];
430 jnrlistC = jjnr[jidx+2];
431 jnrlistD = jjnr[jidx+3];
432 /* Sign of each element will be negative for non-real atoms.
433 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
434 * so use it as val = _mm_andnot_pd(mask,val) to clear dummy entries.
436 tmpmask0 = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
438 tmpmask1 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(3,3,2,2));
439 tmpmask0 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(1,1,0,0));
440 dummy_mask = _mm256_castps_pd(gmx_mm256_set_m128(tmpmask1,tmpmask0));
442 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
443 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
444 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
445 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
446 j_coord_offsetA = DIM*jnrA;
447 j_coord_offsetB = DIM*jnrB;
448 j_coord_offsetC = DIM*jnrC;
449 j_coord_offsetD = DIM*jnrD;
451 /* load j atom coordinates */
452 gmx_mm256_load_1rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
453 x+j_coord_offsetC,x+j_coord_offsetD,
456 /* Calculate displacement vector */
457 dx00 = _mm256_sub_pd(ix0,jx0);
458 dy00 = _mm256_sub_pd(iy0,jy0);
459 dz00 = _mm256_sub_pd(iz0,jz0);
460 dx10 = _mm256_sub_pd(ix1,jx0);
461 dy10 = _mm256_sub_pd(iy1,jy0);
462 dz10 = _mm256_sub_pd(iz1,jz0);
463 dx20 = _mm256_sub_pd(ix2,jx0);
464 dy20 = _mm256_sub_pd(iy2,jy0);
465 dz20 = _mm256_sub_pd(iz2,jz0);
466 dx30 = _mm256_sub_pd(ix3,jx0);
467 dy30 = _mm256_sub_pd(iy3,jy0);
468 dz30 = _mm256_sub_pd(iz3,jz0);
470 /* Calculate squared distance and things based on it */
471 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
472 rsq10 = gmx_mm256_calc_rsq_pd(dx10,dy10,dz10);
473 rsq20 = gmx_mm256_calc_rsq_pd(dx20,dy20,dz20);
474 rsq30 = gmx_mm256_calc_rsq_pd(dx30,dy30,dz30);
476 rinv00 = gmx_mm256_invsqrt_pd(rsq00);
477 rinv10 = gmx_mm256_invsqrt_pd(rsq10);
478 rinv20 = gmx_mm256_invsqrt_pd(rsq20);
479 rinv30 = gmx_mm256_invsqrt_pd(rsq30);
481 rinvsq10 = _mm256_mul_pd(rinv10,rinv10);
482 rinvsq20 = _mm256_mul_pd(rinv20,rinv20);
483 rinvsq30 = _mm256_mul_pd(rinv30,rinv30);
485 /* Load parameters for j particles */
486 jq0 = gmx_mm256_load_4real_swizzle_pd(charge+jnrA+0,charge+jnrB+0,
487 charge+jnrC+0,charge+jnrD+0);
488 vdwjidx0A = 2*vdwtype[jnrA+0];
489 vdwjidx0B = 2*vdwtype[jnrB+0];
490 vdwjidx0C = 2*vdwtype[jnrC+0];
491 vdwjidx0D = 2*vdwtype[jnrD+0];
493 fjx0 = _mm256_setzero_pd();
494 fjy0 = _mm256_setzero_pd();
495 fjz0 = _mm256_setzero_pd();
497 /**************************
498 * CALCULATE INTERACTIONS *
499 **************************/
501 r00 = _mm256_mul_pd(rsq00,rinv00);
502 r00 = _mm256_andnot_pd(dummy_mask,r00);
504 /* Compute parameters for interactions between i and j atoms */
505 gmx_mm256_load_4pair_swizzle_pd(vdwioffsetptr0+vdwjidx0A,
506 vdwioffsetptr0+vdwjidx0B,
507 vdwioffsetptr0+vdwjidx0C,
508 vdwioffsetptr0+vdwjidx0D,
511 /* Calculate table index by multiplying r with table scale and truncate to integer */
512 rt = _mm256_mul_pd(r00,vftabscale);
513 vfitab = _mm256_cvttpd_epi32(rt);
514 vfeps = _mm256_sub_pd(rt,_mm256_round_pd(rt, _MM_FROUND_FLOOR));
515 vfitab = _mm_slli_epi32(vfitab,3);
517 /* CUBIC SPLINE TABLE DISPERSION */
518 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
519 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
520 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
521 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
522 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
523 Heps = _mm256_mul_pd(vfeps,H);
524 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
525 VV = _mm256_add_pd(Y,_mm256_mul_pd(vfeps,Fp));
526 vvdw6 = _mm256_mul_pd(c6_00,VV);
527 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
528 fvdw6 = _mm256_mul_pd(c6_00,FF);
530 /* CUBIC SPLINE TABLE REPULSION */
531 vfitab = _mm_add_epi32(vfitab,ifour);
532 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
533 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
534 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
535 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
536 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
537 Heps = _mm256_mul_pd(vfeps,H);
538 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
539 VV = _mm256_add_pd(Y,_mm256_mul_pd(vfeps,Fp));
540 vvdw12 = _mm256_mul_pd(c12_00,VV);
541 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
542 fvdw12 = _mm256_mul_pd(c12_00,FF);
543 vvdw = _mm256_add_pd(vvdw12,vvdw6);
544 fvdw = _mm256_xor_pd(signbit,_mm256_mul_pd(_mm256_add_pd(fvdw6,fvdw12),_mm256_mul_pd(vftabscale,rinv00)));
546 /* Update potential sum for this i atom from the interaction with this j atom. */
547 vvdw = _mm256_andnot_pd(dummy_mask,vvdw);
548 vvdwsum = _mm256_add_pd(vvdwsum,vvdw);
552 fscal = _mm256_andnot_pd(dummy_mask,fscal);
554 /* Calculate temporary vectorial force */
555 tx = _mm256_mul_pd(fscal,dx00);
556 ty = _mm256_mul_pd(fscal,dy00);
557 tz = _mm256_mul_pd(fscal,dz00);
559 /* Update vectorial force */
560 fix0 = _mm256_add_pd(fix0,tx);
561 fiy0 = _mm256_add_pd(fiy0,ty);
562 fiz0 = _mm256_add_pd(fiz0,tz);
564 fjx0 = _mm256_add_pd(fjx0,tx);
565 fjy0 = _mm256_add_pd(fjy0,ty);
566 fjz0 = _mm256_add_pd(fjz0,tz);
568 /**************************
569 * CALCULATE INTERACTIONS *
570 **************************/
572 /* Compute parameters for interactions between i and j atoms */
573 qq10 = _mm256_mul_pd(iq1,jq0);
575 /* REACTION-FIELD ELECTROSTATICS */
576 velec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_add_pd(rinv10,_mm256_mul_pd(krf,rsq10)),crf));
577 felec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_mul_pd(rinv10,rinvsq10),krf2));
579 /* Update potential sum for this i atom from the interaction with this j atom. */
580 velec = _mm256_andnot_pd(dummy_mask,velec);
581 velecsum = _mm256_add_pd(velecsum,velec);
585 fscal = _mm256_andnot_pd(dummy_mask,fscal);
587 /* Calculate temporary vectorial force */
588 tx = _mm256_mul_pd(fscal,dx10);
589 ty = _mm256_mul_pd(fscal,dy10);
590 tz = _mm256_mul_pd(fscal,dz10);
592 /* Update vectorial force */
593 fix1 = _mm256_add_pd(fix1,tx);
594 fiy1 = _mm256_add_pd(fiy1,ty);
595 fiz1 = _mm256_add_pd(fiz1,tz);
597 fjx0 = _mm256_add_pd(fjx0,tx);
598 fjy0 = _mm256_add_pd(fjy0,ty);
599 fjz0 = _mm256_add_pd(fjz0,tz);
601 /**************************
602 * CALCULATE INTERACTIONS *
603 **************************/
605 /* Compute parameters for interactions between i and j atoms */
606 qq20 = _mm256_mul_pd(iq2,jq0);
608 /* REACTION-FIELD ELECTROSTATICS */
609 velec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_add_pd(rinv20,_mm256_mul_pd(krf,rsq20)),crf));
610 felec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_mul_pd(rinv20,rinvsq20),krf2));
612 /* Update potential sum for this i atom from the interaction with this j atom. */
613 velec = _mm256_andnot_pd(dummy_mask,velec);
614 velecsum = _mm256_add_pd(velecsum,velec);
618 fscal = _mm256_andnot_pd(dummy_mask,fscal);
620 /* Calculate temporary vectorial force */
621 tx = _mm256_mul_pd(fscal,dx20);
622 ty = _mm256_mul_pd(fscal,dy20);
623 tz = _mm256_mul_pd(fscal,dz20);
625 /* Update vectorial force */
626 fix2 = _mm256_add_pd(fix2,tx);
627 fiy2 = _mm256_add_pd(fiy2,ty);
628 fiz2 = _mm256_add_pd(fiz2,tz);
630 fjx0 = _mm256_add_pd(fjx0,tx);
631 fjy0 = _mm256_add_pd(fjy0,ty);
632 fjz0 = _mm256_add_pd(fjz0,tz);
634 /**************************
635 * CALCULATE INTERACTIONS *
636 **************************/
638 /* Compute parameters for interactions between i and j atoms */
639 qq30 = _mm256_mul_pd(iq3,jq0);
641 /* REACTION-FIELD ELECTROSTATICS */
642 velec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_add_pd(rinv30,_mm256_mul_pd(krf,rsq30)),crf));
643 felec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_mul_pd(rinv30,rinvsq30),krf2));
645 /* Update potential sum for this i atom from the interaction with this j atom. */
646 velec = _mm256_andnot_pd(dummy_mask,velec);
647 velecsum = _mm256_add_pd(velecsum,velec);
651 fscal = _mm256_andnot_pd(dummy_mask,fscal);
653 /* Calculate temporary vectorial force */
654 tx = _mm256_mul_pd(fscal,dx30);
655 ty = _mm256_mul_pd(fscal,dy30);
656 tz = _mm256_mul_pd(fscal,dz30);
658 /* Update vectorial force */
659 fix3 = _mm256_add_pd(fix3,tx);
660 fiy3 = _mm256_add_pd(fiy3,ty);
661 fiz3 = _mm256_add_pd(fiz3,tz);
663 fjx0 = _mm256_add_pd(fjx0,tx);
664 fjy0 = _mm256_add_pd(fjy0,ty);
665 fjz0 = _mm256_add_pd(fjz0,tz);
667 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
668 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
669 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
670 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
672 gmx_mm256_decrement_1rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
674 /* Inner loop uses 156 flops */
677 /* End of innermost loop */
679 gmx_mm256_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
680 f+i_coord_offset,fshift+i_shift_offset);
683 /* Update potential energies */
684 gmx_mm256_update_1pot_pd(velecsum,kernel_data->energygrp_elec+ggid);
685 gmx_mm256_update_1pot_pd(vvdwsum,kernel_data->energygrp_vdw+ggid);
687 /* Increment number of inner iterations */
688 inneriter += j_index_end - j_index_start;
690 /* Outer loop uses 26 flops */
693 /* Increment number of outer iterations */
696 /* Update outer/inner flops */
698 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_VF,outeriter*26 + inneriter*156);
701 * Gromacs nonbonded kernel: nb_kernel_ElecRF_VdwCSTab_GeomW4P1_F_avx_256_double
702 * Electrostatics interaction: ReactionField
703 * VdW interaction: CubicSplineTable
704 * Geometry: Water4-Particle
705 * Calculate force/pot: Force
708 nb_kernel_ElecRF_VdwCSTab_GeomW4P1_F_avx_256_double
709 (t_nblist * gmx_restrict nlist,
710 rvec * gmx_restrict xx,
711 rvec * gmx_restrict ff,
712 t_forcerec * gmx_restrict fr,
713 t_mdatoms * gmx_restrict mdatoms,
714 nb_kernel_data_t gmx_unused * gmx_restrict kernel_data,
715 t_nrnb * gmx_restrict nrnb)
717 /* Suffixes 0,1,2,3 refer to particle indices for waters in the inner or outer loop, or
718 * just 0 for non-waters.
719 * Suffixes A,B,C,D refer to j loop unrolling done with AVX, e.g. for the four different
720 * jnr indices corresponding to data put in the four positions in the SIMD register.
722 int i_shift_offset,i_coord_offset,outeriter,inneriter;
723 int j_index_start,j_index_end,jidx,nri,inr,ggid,iidx;
724 int jnrA,jnrB,jnrC,jnrD;
725 int jnrlistA,jnrlistB,jnrlistC,jnrlistD;
726 int jnrlistE,jnrlistF,jnrlistG,jnrlistH;
727 int j_coord_offsetA,j_coord_offsetB,j_coord_offsetC,j_coord_offsetD;
728 int *iinr,*jindex,*jjnr,*shiftidx,*gid;
730 real *shiftvec,*fshift,*x,*f;
731 real *fjptrA,*fjptrB,*fjptrC,*fjptrD;
733 __m256d tx,ty,tz,fscal,rcutoff,rcutoff2,jidxall;
734 real * vdwioffsetptr0;
735 __m256d ix0,iy0,iz0,fix0,fiy0,fiz0,iq0,isai0;
736 real * vdwioffsetptr1;
737 __m256d ix1,iy1,iz1,fix1,fiy1,fiz1,iq1,isai1;
738 real * vdwioffsetptr2;
739 __m256d ix2,iy2,iz2,fix2,fiy2,fiz2,iq2,isai2;
740 real * vdwioffsetptr3;
741 __m256d ix3,iy3,iz3,fix3,fiy3,fiz3,iq3,isai3;
742 int vdwjidx0A,vdwjidx0B,vdwjidx0C,vdwjidx0D;
743 __m256d jx0,jy0,jz0,fjx0,fjy0,fjz0,jq0,isaj0;
744 __m256d dx00,dy00,dz00,rsq00,rinv00,rinvsq00,r00,qq00,c6_00,c12_00;
745 __m256d dx10,dy10,dz10,rsq10,rinv10,rinvsq10,r10,qq10,c6_10,c12_10;
746 __m256d dx20,dy20,dz20,rsq20,rinv20,rinvsq20,r20,qq20,c6_20,c12_20;
747 __m256d dx30,dy30,dz30,rsq30,rinv30,rinvsq30,r30,qq30,c6_30,c12_30;
748 __m256d velec,felec,velecsum,facel,crf,krf,krf2;
751 __m256d rinvsix,rvdw,vvdw,vvdw6,vvdw12,fvdw,fvdw6,fvdw12,vvdwsum,sh_vdw_invrcut6;
754 __m256d one_sixth = _mm256_set1_pd(1.0/6.0);
755 __m256d one_twelfth = _mm256_set1_pd(1.0/12.0);
757 __m128i ifour = _mm_set1_epi32(4);
758 __m256d rt,vfeps,vftabscale,Y,F,G,H,Heps,Fp,VV,FF;
760 __m256d dummy_mask,cutoff_mask;
761 __m128 tmpmask0,tmpmask1;
762 __m256d signbit = _mm256_castsi256_pd( _mm256_set1_epi32(0x80000000) );
763 __m256d one = _mm256_set1_pd(1.0);
764 __m256d two = _mm256_set1_pd(2.0);
770 jindex = nlist->jindex;
772 shiftidx = nlist->shift;
774 shiftvec = fr->shift_vec[0];
775 fshift = fr->fshift[0];
776 facel = _mm256_set1_pd(fr->epsfac);
777 charge = mdatoms->chargeA;
778 krf = _mm256_set1_pd(fr->ic->k_rf);
779 krf2 = _mm256_set1_pd(fr->ic->k_rf*2.0);
780 crf = _mm256_set1_pd(fr->ic->c_rf);
781 nvdwtype = fr->ntype;
783 vdwtype = mdatoms->typeA;
785 vftab = kernel_data->table_vdw->data;
786 vftabscale = _mm256_set1_pd(kernel_data->table_vdw->scale);
788 /* Setup water-specific parameters */
789 inr = nlist->iinr[0];
790 iq1 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+1]));
791 iq2 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+2]));
792 iq3 = _mm256_mul_pd(facel,_mm256_set1_pd(charge[inr+3]));
793 vdwioffsetptr0 = vdwparam+2*nvdwtype*vdwtype[inr+0];
795 /* Avoid stupid compiler warnings */
796 jnrA = jnrB = jnrC = jnrD = 0;
805 for(iidx=0;iidx<4*DIM;iidx++)
810 /* Start outer loop over neighborlists */
811 for(iidx=0; iidx<nri; iidx++)
813 /* Load shift vector for this list */
814 i_shift_offset = DIM*shiftidx[iidx];
816 /* Load limits for loop over neighbors */
817 j_index_start = jindex[iidx];
818 j_index_end = jindex[iidx+1];
820 /* Get outer coordinate index */
822 i_coord_offset = DIM*inr;
824 /* Load i particle coords and add shift vector */
825 gmx_mm256_load_shift_and_4rvec_broadcast_pd(shiftvec+i_shift_offset,x+i_coord_offset,
826 &ix0,&iy0,&iz0,&ix1,&iy1,&iz1,&ix2,&iy2,&iz2,&ix3,&iy3,&iz3);
828 fix0 = _mm256_setzero_pd();
829 fiy0 = _mm256_setzero_pd();
830 fiz0 = _mm256_setzero_pd();
831 fix1 = _mm256_setzero_pd();
832 fiy1 = _mm256_setzero_pd();
833 fiz1 = _mm256_setzero_pd();
834 fix2 = _mm256_setzero_pd();
835 fiy2 = _mm256_setzero_pd();
836 fiz2 = _mm256_setzero_pd();
837 fix3 = _mm256_setzero_pd();
838 fiy3 = _mm256_setzero_pd();
839 fiz3 = _mm256_setzero_pd();
841 /* Start inner kernel loop */
842 for(jidx=j_index_start; jidx<j_index_end && jjnr[jidx+3]>=0; jidx+=4)
845 /* Get j neighbor index, and coordinate index */
850 j_coord_offsetA = DIM*jnrA;
851 j_coord_offsetB = DIM*jnrB;
852 j_coord_offsetC = DIM*jnrC;
853 j_coord_offsetD = DIM*jnrD;
855 /* load j atom coordinates */
856 gmx_mm256_load_1rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
857 x+j_coord_offsetC,x+j_coord_offsetD,
860 /* Calculate displacement vector */
861 dx00 = _mm256_sub_pd(ix0,jx0);
862 dy00 = _mm256_sub_pd(iy0,jy0);
863 dz00 = _mm256_sub_pd(iz0,jz0);
864 dx10 = _mm256_sub_pd(ix1,jx0);
865 dy10 = _mm256_sub_pd(iy1,jy0);
866 dz10 = _mm256_sub_pd(iz1,jz0);
867 dx20 = _mm256_sub_pd(ix2,jx0);
868 dy20 = _mm256_sub_pd(iy2,jy0);
869 dz20 = _mm256_sub_pd(iz2,jz0);
870 dx30 = _mm256_sub_pd(ix3,jx0);
871 dy30 = _mm256_sub_pd(iy3,jy0);
872 dz30 = _mm256_sub_pd(iz3,jz0);
874 /* Calculate squared distance and things based on it */
875 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
876 rsq10 = gmx_mm256_calc_rsq_pd(dx10,dy10,dz10);
877 rsq20 = gmx_mm256_calc_rsq_pd(dx20,dy20,dz20);
878 rsq30 = gmx_mm256_calc_rsq_pd(dx30,dy30,dz30);
880 rinv00 = gmx_mm256_invsqrt_pd(rsq00);
881 rinv10 = gmx_mm256_invsqrt_pd(rsq10);
882 rinv20 = gmx_mm256_invsqrt_pd(rsq20);
883 rinv30 = gmx_mm256_invsqrt_pd(rsq30);
885 rinvsq10 = _mm256_mul_pd(rinv10,rinv10);
886 rinvsq20 = _mm256_mul_pd(rinv20,rinv20);
887 rinvsq30 = _mm256_mul_pd(rinv30,rinv30);
889 /* Load parameters for j particles */
890 jq0 = gmx_mm256_load_4real_swizzle_pd(charge+jnrA+0,charge+jnrB+0,
891 charge+jnrC+0,charge+jnrD+0);
892 vdwjidx0A = 2*vdwtype[jnrA+0];
893 vdwjidx0B = 2*vdwtype[jnrB+0];
894 vdwjidx0C = 2*vdwtype[jnrC+0];
895 vdwjidx0D = 2*vdwtype[jnrD+0];
897 fjx0 = _mm256_setzero_pd();
898 fjy0 = _mm256_setzero_pd();
899 fjz0 = _mm256_setzero_pd();
901 /**************************
902 * CALCULATE INTERACTIONS *
903 **************************/
905 r00 = _mm256_mul_pd(rsq00,rinv00);
907 /* Compute parameters for interactions between i and j atoms */
908 gmx_mm256_load_4pair_swizzle_pd(vdwioffsetptr0+vdwjidx0A,
909 vdwioffsetptr0+vdwjidx0B,
910 vdwioffsetptr0+vdwjidx0C,
911 vdwioffsetptr0+vdwjidx0D,
914 /* Calculate table index by multiplying r with table scale and truncate to integer */
915 rt = _mm256_mul_pd(r00,vftabscale);
916 vfitab = _mm256_cvttpd_epi32(rt);
917 vfeps = _mm256_sub_pd(rt,_mm256_round_pd(rt, _MM_FROUND_FLOOR));
918 vfitab = _mm_slli_epi32(vfitab,3);
920 /* CUBIC SPLINE TABLE DISPERSION */
921 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
922 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
923 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
924 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
925 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
926 Heps = _mm256_mul_pd(vfeps,H);
927 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
928 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
929 fvdw6 = _mm256_mul_pd(c6_00,FF);
931 /* CUBIC SPLINE TABLE REPULSION */
932 vfitab = _mm_add_epi32(vfitab,ifour);
933 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
934 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
935 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
936 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
937 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
938 Heps = _mm256_mul_pd(vfeps,H);
939 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
940 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
941 fvdw12 = _mm256_mul_pd(c12_00,FF);
942 fvdw = _mm256_xor_pd(signbit,_mm256_mul_pd(_mm256_add_pd(fvdw6,fvdw12),_mm256_mul_pd(vftabscale,rinv00)));
946 /* Calculate temporary vectorial force */
947 tx = _mm256_mul_pd(fscal,dx00);
948 ty = _mm256_mul_pd(fscal,dy00);
949 tz = _mm256_mul_pd(fscal,dz00);
951 /* Update vectorial force */
952 fix0 = _mm256_add_pd(fix0,tx);
953 fiy0 = _mm256_add_pd(fiy0,ty);
954 fiz0 = _mm256_add_pd(fiz0,tz);
956 fjx0 = _mm256_add_pd(fjx0,tx);
957 fjy0 = _mm256_add_pd(fjy0,ty);
958 fjz0 = _mm256_add_pd(fjz0,tz);
960 /**************************
961 * CALCULATE INTERACTIONS *
962 **************************/
964 /* Compute parameters for interactions between i and j atoms */
965 qq10 = _mm256_mul_pd(iq1,jq0);
967 /* REACTION-FIELD ELECTROSTATICS */
968 felec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_mul_pd(rinv10,rinvsq10),krf2));
972 /* Calculate temporary vectorial force */
973 tx = _mm256_mul_pd(fscal,dx10);
974 ty = _mm256_mul_pd(fscal,dy10);
975 tz = _mm256_mul_pd(fscal,dz10);
977 /* Update vectorial force */
978 fix1 = _mm256_add_pd(fix1,tx);
979 fiy1 = _mm256_add_pd(fiy1,ty);
980 fiz1 = _mm256_add_pd(fiz1,tz);
982 fjx0 = _mm256_add_pd(fjx0,tx);
983 fjy0 = _mm256_add_pd(fjy0,ty);
984 fjz0 = _mm256_add_pd(fjz0,tz);
986 /**************************
987 * CALCULATE INTERACTIONS *
988 **************************/
990 /* Compute parameters for interactions between i and j atoms */
991 qq20 = _mm256_mul_pd(iq2,jq0);
993 /* REACTION-FIELD ELECTROSTATICS */
994 felec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_mul_pd(rinv20,rinvsq20),krf2));
998 /* Calculate temporary vectorial force */
999 tx = _mm256_mul_pd(fscal,dx20);
1000 ty = _mm256_mul_pd(fscal,dy20);
1001 tz = _mm256_mul_pd(fscal,dz20);
1003 /* Update vectorial force */
1004 fix2 = _mm256_add_pd(fix2,tx);
1005 fiy2 = _mm256_add_pd(fiy2,ty);
1006 fiz2 = _mm256_add_pd(fiz2,tz);
1008 fjx0 = _mm256_add_pd(fjx0,tx);
1009 fjy0 = _mm256_add_pd(fjy0,ty);
1010 fjz0 = _mm256_add_pd(fjz0,tz);
1012 /**************************
1013 * CALCULATE INTERACTIONS *
1014 **************************/
1016 /* Compute parameters for interactions between i and j atoms */
1017 qq30 = _mm256_mul_pd(iq3,jq0);
1019 /* REACTION-FIELD ELECTROSTATICS */
1020 felec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_mul_pd(rinv30,rinvsq30),krf2));
1024 /* Calculate temporary vectorial force */
1025 tx = _mm256_mul_pd(fscal,dx30);
1026 ty = _mm256_mul_pd(fscal,dy30);
1027 tz = _mm256_mul_pd(fscal,dz30);
1029 /* Update vectorial force */
1030 fix3 = _mm256_add_pd(fix3,tx);
1031 fiy3 = _mm256_add_pd(fiy3,ty);
1032 fiz3 = _mm256_add_pd(fiz3,tz);
1034 fjx0 = _mm256_add_pd(fjx0,tx);
1035 fjy0 = _mm256_add_pd(fjy0,ty);
1036 fjz0 = _mm256_add_pd(fjz0,tz);
1038 fjptrA = f+j_coord_offsetA;
1039 fjptrB = f+j_coord_offsetB;
1040 fjptrC = f+j_coord_offsetC;
1041 fjptrD = f+j_coord_offsetD;
1043 gmx_mm256_decrement_1rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1045 /* Inner loop uses 132 flops */
1048 if(jidx<j_index_end)
1051 /* Get j neighbor index, and coordinate index */
1052 jnrlistA = jjnr[jidx];
1053 jnrlistB = jjnr[jidx+1];
1054 jnrlistC = jjnr[jidx+2];
1055 jnrlistD = jjnr[jidx+3];
1056 /* Sign of each element will be negative for non-real atoms.
1057 * This mask will be 0xFFFFFFFF for dummy entries and 0x0 for real ones,
1058 * so use it as val = _mm_andnot_pd(mask,val) to clear dummy entries.
1060 tmpmask0 = gmx_mm_castsi128_ps(_mm_cmplt_epi32(_mm_loadu_si128((const __m128i *)(jjnr+jidx)),_mm_setzero_si128()));
1062 tmpmask1 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(3,3,2,2));
1063 tmpmask0 = _mm_permute_ps(tmpmask0,_GMX_MM_PERMUTE(1,1,0,0));
1064 dummy_mask = _mm256_castps_pd(gmx_mm256_set_m128(tmpmask1,tmpmask0));
1066 jnrA = (jnrlistA>=0) ? jnrlistA : 0;
1067 jnrB = (jnrlistB>=0) ? jnrlistB : 0;
1068 jnrC = (jnrlistC>=0) ? jnrlistC : 0;
1069 jnrD = (jnrlistD>=0) ? jnrlistD : 0;
1070 j_coord_offsetA = DIM*jnrA;
1071 j_coord_offsetB = DIM*jnrB;
1072 j_coord_offsetC = DIM*jnrC;
1073 j_coord_offsetD = DIM*jnrD;
1075 /* load j atom coordinates */
1076 gmx_mm256_load_1rvec_4ptr_swizzle_pd(x+j_coord_offsetA,x+j_coord_offsetB,
1077 x+j_coord_offsetC,x+j_coord_offsetD,
1080 /* Calculate displacement vector */
1081 dx00 = _mm256_sub_pd(ix0,jx0);
1082 dy00 = _mm256_sub_pd(iy0,jy0);
1083 dz00 = _mm256_sub_pd(iz0,jz0);
1084 dx10 = _mm256_sub_pd(ix1,jx0);
1085 dy10 = _mm256_sub_pd(iy1,jy0);
1086 dz10 = _mm256_sub_pd(iz1,jz0);
1087 dx20 = _mm256_sub_pd(ix2,jx0);
1088 dy20 = _mm256_sub_pd(iy2,jy0);
1089 dz20 = _mm256_sub_pd(iz2,jz0);
1090 dx30 = _mm256_sub_pd(ix3,jx0);
1091 dy30 = _mm256_sub_pd(iy3,jy0);
1092 dz30 = _mm256_sub_pd(iz3,jz0);
1094 /* Calculate squared distance and things based on it */
1095 rsq00 = gmx_mm256_calc_rsq_pd(dx00,dy00,dz00);
1096 rsq10 = gmx_mm256_calc_rsq_pd(dx10,dy10,dz10);
1097 rsq20 = gmx_mm256_calc_rsq_pd(dx20,dy20,dz20);
1098 rsq30 = gmx_mm256_calc_rsq_pd(dx30,dy30,dz30);
1100 rinv00 = gmx_mm256_invsqrt_pd(rsq00);
1101 rinv10 = gmx_mm256_invsqrt_pd(rsq10);
1102 rinv20 = gmx_mm256_invsqrt_pd(rsq20);
1103 rinv30 = gmx_mm256_invsqrt_pd(rsq30);
1105 rinvsq10 = _mm256_mul_pd(rinv10,rinv10);
1106 rinvsq20 = _mm256_mul_pd(rinv20,rinv20);
1107 rinvsq30 = _mm256_mul_pd(rinv30,rinv30);
1109 /* Load parameters for j particles */
1110 jq0 = gmx_mm256_load_4real_swizzle_pd(charge+jnrA+0,charge+jnrB+0,
1111 charge+jnrC+0,charge+jnrD+0);
1112 vdwjidx0A = 2*vdwtype[jnrA+0];
1113 vdwjidx0B = 2*vdwtype[jnrB+0];
1114 vdwjidx0C = 2*vdwtype[jnrC+0];
1115 vdwjidx0D = 2*vdwtype[jnrD+0];
1117 fjx0 = _mm256_setzero_pd();
1118 fjy0 = _mm256_setzero_pd();
1119 fjz0 = _mm256_setzero_pd();
1121 /**************************
1122 * CALCULATE INTERACTIONS *
1123 **************************/
1125 r00 = _mm256_mul_pd(rsq00,rinv00);
1126 r00 = _mm256_andnot_pd(dummy_mask,r00);
1128 /* Compute parameters for interactions between i and j atoms */
1129 gmx_mm256_load_4pair_swizzle_pd(vdwioffsetptr0+vdwjidx0A,
1130 vdwioffsetptr0+vdwjidx0B,
1131 vdwioffsetptr0+vdwjidx0C,
1132 vdwioffsetptr0+vdwjidx0D,
1135 /* Calculate table index by multiplying r with table scale and truncate to integer */
1136 rt = _mm256_mul_pd(r00,vftabscale);
1137 vfitab = _mm256_cvttpd_epi32(rt);
1138 vfeps = _mm256_sub_pd(rt,_mm256_round_pd(rt, _MM_FROUND_FLOOR));
1139 vfitab = _mm_slli_epi32(vfitab,3);
1141 /* CUBIC SPLINE TABLE DISPERSION */
1142 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1143 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1144 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
1145 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
1146 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
1147 Heps = _mm256_mul_pd(vfeps,H);
1148 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
1149 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
1150 fvdw6 = _mm256_mul_pd(c6_00,FF);
1152 /* CUBIC SPLINE TABLE REPULSION */
1153 vfitab = _mm_add_epi32(vfitab,ifour);
1154 Y = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,0) );
1155 F = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,1) );
1156 G = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,2) );
1157 H = _mm256_load_pd( vftab + _mm_extract_epi32(vfitab,3) );
1158 GMX_MM256_FULLTRANSPOSE4_PD(Y,F,G,H);
1159 Heps = _mm256_mul_pd(vfeps,H);
1160 Fp = _mm256_add_pd(F,_mm256_mul_pd(vfeps,_mm256_add_pd(G,Heps)));
1161 FF = _mm256_add_pd(Fp,_mm256_mul_pd(vfeps,_mm256_add_pd(G,_mm256_add_pd(Heps,Heps))));
1162 fvdw12 = _mm256_mul_pd(c12_00,FF);
1163 fvdw = _mm256_xor_pd(signbit,_mm256_mul_pd(_mm256_add_pd(fvdw6,fvdw12),_mm256_mul_pd(vftabscale,rinv00)));
1167 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1169 /* Calculate temporary vectorial force */
1170 tx = _mm256_mul_pd(fscal,dx00);
1171 ty = _mm256_mul_pd(fscal,dy00);
1172 tz = _mm256_mul_pd(fscal,dz00);
1174 /* Update vectorial force */
1175 fix0 = _mm256_add_pd(fix0,tx);
1176 fiy0 = _mm256_add_pd(fiy0,ty);
1177 fiz0 = _mm256_add_pd(fiz0,tz);
1179 fjx0 = _mm256_add_pd(fjx0,tx);
1180 fjy0 = _mm256_add_pd(fjy0,ty);
1181 fjz0 = _mm256_add_pd(fjz0,tz);
1183 /**************************
1184 * CALCULATE INTERACTIONS *
1185 **************************/
1187 /* Compute parameters for interactions between i and j atoms */
1188 qq10 = _mm256_mul_pd(iq1,jq0);
1190 /* REACTION-FIELD ELECTROSTATICS */
1191 felec = _mm256_mul_pd(qq10,_mm256_sub_pd(_mm256_mul_pd(rinv10,rinvsq10),krf2));
1195 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1197 /* Calculate temporary vectorial force */
1198 tx = _mm256_mul_pd(fscal,dx10);
1199 ty = _mm256_mul_pd(fscal,dy10);
1200 tz = _mm256_mul_pd(fscal,dz10);
1202 /* Update vectorial force */
1203 fix1 = _mm256_add_pd(fix1,tx);
1204 fiy1 = _mm256_add_pd(fiy1,ty);
1205 fiz1 = _mm256_add_pd(fiz1,tz);
1207 fjx0 = _mm256_add_pd(fjx0,tx);
1208 fjy0 = _mm256_add_pd(fjy0,ty);
1209 fjz0 = _mm256_add_pd(fjz0,tz);
1211 /**************************
1212 * CALCULATE INTERACTIONS *
1213 **************************/
1215 /* Compute parameters for interactions between i and j atoms */
1216 qq20 = _mm256_mul_pd(iq2,jq0);
1218 /* REACTION-FIELD ELECTROSTATICS */
1219 felec = _mm256_mul_pd(qq20,_mm256_sub_pd(_mm256_mul_pd(rinv20,rinvsq20),krf2));
1223 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1225 /* Calculate temporary vectorial force */
1226 tx = _mm256_mul_pd(fscal,dx20);
1227 ty = _mm256_mul_pd(fscal,dy20);
1228 tz = _mm256_mul_pd(fscal,dz20);
1230 /* Update vectorial force */
1231 fix2 = _mm256_add_pd(fix2,tx);
1232 fiy2 = _mm256_add_pd(fiy2,ty);
1233 fiz2 = _mm256_add_pd(fiz2,tz);
1235 fjx0 = _mm256_add_pd(fjx0,tx);
1236 fjy0 = _mm256_add_pd(fjy0,ty);
1237 fjz0 = _mm256_add_pd(fjz0,tz);
1239 /**************************
1240 * CALCULATE INTERACTIONS *
1241 **************************/
1243 /* Compute parameters for interactions between i and j atoms */
1244 qq30 = _mm256_mul_pd(iq3,jq0);
1246 /* REACTION-FIELD ELECTROSTATICS */
1247 felec = _mm256_mul_pd(qq30,_mm256_sub_pd(_mm256_mul_pd(rinv30,rinvsq30),krf2));
1251 fscal = _mm256_andnot_pd(dummy_mask,fscal);
1253 /* Calculate temporary vectorial force */
1254 tx = _mm256_mul_pd(fscal,dx30);
1255 ty = _mm256_mul_pd(fscal,dy30);
1256 tz = _mm256_mul_pd(fscal,dz30);
1258 /* Update vectorial force */
1259 fix3 = _mm256_add_pd(fix3,tx);
1260 fiy3 = _mm256_add_pd(fiy3,ty);
1261 fiz3 = _mm256_add_pd(fiz3,tz);
1263 fjx0 = _mm256_add_pd(fjx0,tx);
1264 fjy0 = _mm256_add_pd(fjy0,ty);
1265 fjz0 = _mm256_add_pd(fjz0,tz);
1267 fjptrA = (jnrlistA>=0) ? f+j_coord_offsetA : scratch;
1268 fjptrB = (jnrlistB>=0) ? f+j_coord_offsetB : scratch;
1269 fjptrC = (jnrlistC>=0) ? f+j_coord_offsetC : scratch;
1270 fjptrD = (jnrlistD>=0) ? f+j_coord_offsetD : scratch;
1272 gmx_mm256_decrement_1rvec_4ptr_swizzle_pd(fjptrA,fjptrB,fjptrC,fjptrD,fjx0,fjy0,fjz0);
1274 /* Inner loop uses 133 flops */
1277 /* End of innermost loop */
1279 gmx_mm256_update_iforce_4atom_swizzle_pd(fix0,fiy0,fiz0,fix1,fiy1,fiz1,fix2,fiy2,fiz2,fix3,fiy3,fiz3,
1280 f+i_coord_offset,fshift+i_shift_offset);
1282 /* Increment number of inner iterations */
1283 inneriter += j_index_end - j_index_start;
1285 /* Outer loop uses 24 flops */
1288 /* Increment number of outer iterations */
1291 /* Update outer/inner flops */
1293 inc_nrnb(nrnb,eNR_NBKERNEL_ELEC_VDW_W4_F,outeriter*24 + inneriter*133);