src/gromacs/mdlib/nbnxn_kernels/nbnxn_kernel_x86_simd256.c

   1 /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
   2  *
   3  *
   4  *                This source code is part of
   5  *
   6  *                 G   R   O   M   A   C   S
   7  *
   8  *          GROningen MAchine for Chemical Simulations
   9  *
  10  * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
  11  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
  12  * Copyright (c) 2001-2012, The GROMACS development team,
  13  * check out http://www.gromacs.org for more information.
  14  *
  15  * This program is free software; you can redistribute it and/or
  16  * modify it under the terms of the GNU General Public License
  17  * as published by the Free Software Foundation; either version 2
  18  * of the License, or (at your option) any later version.
  19  *
  20  * If you want to redistribute modifications, please consider that
  21  * scientific software is very special. Version control is crucial -
  22  * bugs must be traceable. We will be happy to consider code for
  23  * inclusion in the official distribution, but derived work must not
  24  * be called official GROMACS. Details are found in the README & COPYING
  25  * files - if they are missing, get the official version at www.gromacs.org.
  26  *
  27  * To help us fund GROMACS development, we humbly ask that you cite
  28  * the papers on the package - you can find them in the top README file.
  29  *
  30  * For more info, check our website at http://www.gromacs.org
  31  *
  32  * And Hey:
  33  * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
  34  */
  35 #ifdef HAVE_CONFIG_H
  36 #include <config.h>
  37 #endif
  38
  39 #include <math.h>
  40
  41 #include "typedefs.h"
  42 #include "vec.h"
  43 #include "smalloc.h"
  44 #include "force.h"
  45 #include "gmx_omp_nthreads.h"
  46 #include "../nbnxn_consts.h"
  47 #include "nbnxn_kernel_common.h"
  48
  49 #ifdef GMX_X86_AVX_256
  50
  51 #include "nbnxn_kernel_x86_simd256.h"
  52
  53 /* Include all flavors of the 256-bit AVX kernel loops */
  54
  55 #define GMX_MM256_HERE
  56
  57 /* Analytical reaction-field kernels */
  58 #define CALC_COUL_RF
  59
  60 #include "nbnxn_kernel_x86_simd_includes.h"
  61
  62 #undef CALC_COUL_RF
  63
  64 /* Tabulated exclusion interaction electrostatics kernels */
  65 #define CALC_COUL_TAB
  66
  67 /* Single cut-off: rcoulomb = rvdw */
  68 #include "nbnxn_kernel_x86_simd_includes.h"
  69
  70 /* Twin cut-off: rcoulomb >= rvdw */
  71 #define VDW_CUTOFF_CHECK
  72 #include "nbnxn_kernel_x86_simd_includes.h"
  73 #undef VDW_CUTOFF_CHECK
  74
  75 #undef CALC_COUL_TAB
  76
  77 /* Analytical Ewald exclusion interaction electrostatics kernels */
  78 #define CALC_COUL_EWALD
  79
  80 /* Single cut-off: rcoulomb = rvdw */
  81 #include "nbnxn_kernel_x86_simd_includes.h"
  82
  83 /* Twin cut-off: rcoulomb >= rvdw */
  84 #define VDW_CUTOFF_CHECK
  85 #include "nbnxn_kernel_x86_simd_includes.h"
  86 #undef VDW_CUTOFF_CHECK
  87
  88 #undef CALC_COUL_EWALD
  89
  90
  91 typedef void (*p_nbk_func_ener)(const nbnxn_pairlist_t     *nbl,
  92                                 const nbnxn_atomdata_t     *nbat,
  93                                 const interaction_const_t  *ic,
  94                                 rvec                       *shift_vec,
  95                                 real                       *f,
  96                                 real                       *fshift,
  97                                 real                       *Vvdw,
  98                                 real                       *Vc);
  99
 100 typedef void (*p_nbk_func_noener)(const nbnxn_pairlist_t     *nbl,
 101                                   const nbnxn_atomdata_t     *nbat,
 102                                   const interaction_const_t  *ic,
 103                                   rvec                       *shift_vec,
 104                                   real                       *f,
 105                                   real                       *fshift);
 106
 107 enum { coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR };
 108
 109 #define NBK_FN(elec,ljcomb) nbnxn_kernel_x86_simd256_##elec##_comb_##ljcomb##_ener
 110 static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
 111 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 112   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 113   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 114   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 115   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 116 #undef NBK_FN
 117
 118 #define NBK_FN(elec,ljcomb) nbnxn_kernel_x86_simd256_##elec##_comb_##ljcomb##_energrp
 119 static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
 120 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 121   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 122   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 123   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 124   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 125 #undef NBK_FN
 126
 127 #define NBK_FN(elec,ljcomb) nbnxn_kernel_x86_simd256_##elec##_comb_##ljcomb##_noener
 128 static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
 129 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 130   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 131   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 132   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 133   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 134 #undef NBK_FN
 135
 136
 137 static void reduce_group_energies(int ng,int ng_2log,
 138                                   const real *VSvdw,const real *VSc,
 139                                   real *Vvdw,real *Vc)
 140 {
 141     int ng_p2,i,j,j0,j1,c,s;
 142
 143 #define SIMD_WIDTH       (GMX_X86_SIMD_WIDTH_HERE)
 144 #define SIMD_WIDTH_HALF  (GMX_X86_SIMD_WIDTH_HERE/2)
 145
 146     ng_p2 = (1<<ng_2log);
 147
 148     /* The size of the x86 SIMD energy group buffer array is:
 149      * ng*ng*ng_p2*SIMD_WIDTH_HALF*SIMD_WIDTH
 150      */
 151     for(i=0; i<ng; i++)
 152     {
 153         for(j=0; j<ng; j++)
 154         {
 155             Vvdw[i*ng+j] = 0;
 156             Vc[i*ng+j]   = 0;
 157         }
 158
 159         for(j1=0; j1<ng; j1++)
 160         {
 161             for(j0=0; j0<ng; j0++)
 162             {
 163                 c = ((i*ng + j1)*ng_p2 + j0)*SIMD_WIDTH_HALF*SIMD_WIDTH;
 164                 for(s=0; s<SIMD_WIDTH_HALF; s++)
 165                 {
 166                     Vvdw[i*ng+j0] += VSvdw[c+0];
 167                     Vvdw[i*ng+j1] += VSvdw[c+1];
 168                     Vc  [i*ng+j0] += VSc  [c+0];
 169                     Vc  [i*ng+j1] += VSc  [c+1];
 170                     c += SIMD_WIDTH + 2;
 171                 }
 172             }
 173         }
 174     }
 175 }
 176
 177 #endif /* GMX_X86_AVX_256 */
 178
 179 void
 180 nbnxn_kernel_x86_simd256(nbnxn_pairlist_set_t       *nbl_list,
 181                          const nbnxn_atomdata_t     *nbat,
 182                          const interaction_const_t  *ic,
 183                          int                        ewald_excl,
 184                          rvec                       *shift_vec,
 185                          int                        force_flags,
 186                          int                        clearF,
 187                          real                       *fshift,
 188                          real                       *Vc,
 189                          real                       *Vvdw)
 190 #ifdef GMX_X86_AVX_256
 191 {
 192     int              nnbl;
 193     nbnxn_pairlist_t **nbl;
 194     int coult;
 195     int nb;
 196
 197     nnbl = nbl_list->nnbl;
 198     nbl  = nbl_list->nbl;
 199
 200     if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
 201     {
 202         coult = coultRF;
 203     }
 204     else
 205     {
 206         if (ewald_excl == ewaldexclTable)
 207         {
 208             if (ic->rcoulomb == ic->rvdw)
 209             {
 210                 coult = coultTAB;
 211             }
 212             else
 213             {
 214                 coult = coultTAB_TWIN;
 215             }
 216         }
 217         else
 218         {
 219             if (ic->rcoulomb == ic->rvdw)
 220             {
 221                 coult = coultEWALD;
 222             }
 223             else
 224             {
 225                 coult = coultEWALD_TWIN;
 226             }
 227         }
 228     }
 229
 230 #pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
 231     for(nb=0; nb<nnbl; nb++)
 232     {
 233         nbnxn_atomdata_output_t *out;
 234         real *fshift_p;
 235
 236         out = &nbat->out[nb];
 237
 238         if (clearF == enbvClearFYes)
 239         {
 240             clear_f(nbat,out->f);
 241         }
 242
 243         if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
 244         {
 245             fshift_p = fshift;
 246         }
 247         else
 248         {
 249             fshift_p = out->fshift;
 250
 251             if (clearF == enbvClearFYes)
 252             {
 253                 clear_fshift(fshift_p);
 254             }
 255         }
 256
 257         /* With Ewald type electrostatics we the forces for excluded atom pairs
 258          * should not contribute to the virial sum. The exclusion forces
 259          * are not calculate in the energy kernels, but are in _noener.
 260          */
 261         if (!((force_flags & GMX_FORCE_ENERGY) ||
 262               (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
 263         {
 264             /* Don't calculate energies */
 265             p_nbk_noener[coult][nbat->comb_rule](nbl[nb],nbat,
 266                                                  ic,
 267                                                  shift_vec,
 268                                                  out->f,
 269                                                  fshift_p);
 270         }
 271         else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
 272         {
 273             /* No energy groups */
 274             out->Vvdw[0] = 0;
 275             out->Vc[0]   = 0;
 276
 277             p_nbk_ener[coult][nbat->comb_rule](nbl[nb],nbat,
 278                                                ic,
 279                                                shift_vec,
 280                                                out->f,
 281                                                fshift_p,
 282                                                out->Vvdw,
 283                                                out->Vc);
 284         }
 285         else
 286         {
 287             /* Calculate energy group contributions */
 288             int i;
 289
 290             for(i=0; i<out->nVS; i++)
 291             {
 292                 out->VSvdw[i] = 0;
 293             }
 294             for(i=0; i<out->nVS; i++)
 295             {
 296                 out->VSc[i] = 0;
 297             }
 298
 299             p_nbk_energrp[coult][nbat->comb_rule](nbl[nb],nbat,
 300                                                   ic,
 301                                                   shift_vec,
 302                                                   out->f,
 303                                                   fshift_p,
 304                                                   out->VSvdw,
 305                                                   out->VSc);
 306
 307             reduce_group_energies(nbat->nenergrp,nbat->neg_2log,
 308                                   out->VSvdw,out->VSc,
 309                                   out->Vvdw,out->Vc);
 310         }
 311     }
 312
 313     if (force_flags & GMX_FORCE_ENERGY)
 314     {
 315         reduce_energies_over_lists(nbat,nnbl,Vvdw,Vc);
 316     }
 317 }
 318 #else
 319 {
 320     gmx_incons("nbnxn_kernel_x86_simd256 called while GROMACS was configured without AVX enabled");
 321 }
 322 #endif