src/mdlib/nbnxn_kernels/nbnxn_kernel_simd_2xnn.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2012, The GROMACS development team,
   6  * check out http://www.gromacs.org for more information.
   7  * Copyright (c) 2012,2013, by the GROMACS development team, led by
   8  * David van der Spoel, Berk Hess, Erik Lindahl, and including many
   9  * others, as listed in the AUTHORS file in the top-level source
  10  * directory and at http://www.gromacs.org.
  11  *
  12  * GROMACS is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU Lesser General Public License
  14  * as published by the Free Software Foundation; either version 2.1
  15  * of the License, or (at your option) any later version.
  16  *
  17  * GROMACS is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20  * Lesser General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public
  23  * License along with GROMACS; if not, see
  24  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  25  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  26  *
  27  * If you want to redistribute modifications to GROMACS, please
  28  * consider that scientific software is very special. Version
  29  * control is crucial - bugs must be traceable. We will be happy to
  30  * consider code for inclusion in the official distribution, but
  31  * derived work must not be called official GROMACS. Details are found
  32  * in the README & COPYING files - if they are missing, get the
  33  * official version at http://www.gromacs.org.
  34  *
  35  * To help us fund GROMACS development, we humbly ask that you cite
  36  * the research papers on the package. Check out http://www.gromacs.org.
  37  */
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42 #include <math.h>
  43
  44 #include "typedefs.h"
  45 #include "vec.h"
  46 #include "smalloc.h"
  47 #include "force.h"
  48 #include "gmx_omp_nthreads.h"
  49 #include "../nbnxn_consts.h"
  50 #include "nbnxn_kernel_common.h"
  51
  52 #ifdef GMX_NBNXN_SIMD_2XNN
  53
  54 #include "nbnxn_kernel_simd_2xnn.h"
  55
  56 /* Include all flavors of the SSE or AVX 2x(N+N) kernel loops */
  57
  58 #if GMX_NBNXN_SIMD_BITWIDTH == 128
  59 #define GMX_MM128_HERE
  60 #else
  61 #if GMX_NBNXN_SIMD_BITWIDTH == 256
  62 #define GMX_MM256_HERE
  63 #else
  64 #error "unsupported GMX_NBNXN_SIMD_BITWIDTH"
  65 #endif
  66 #endif
  67
  68 /* Analytical reaction-field kernels */
  69 #define CALC_COUL_RF
  70
  71 #include "nbnxn_kernel_simd_2xnn_includes.h"
  72
  73 #undef CALC_COUL_RF
  74
  75 /* Tabulated exclusion interaction electrostatics kernels */
  76 #define CALC_COUL_TAB
  77
  78 /* Single cut-off: rcoulomb = rvdw */
  79 #include "nbnxn_kernel_simd_2xnn_includes.h"
  80
  81 /* Twin cut-off: rcoulomb >= rvdw */
  82 #define VDW_CUTOFF_CHECK
  83 #include "nbnxn_kernel_simd_2xnn_includes.h"
  84 #undef VDW_CUTOFF_CHECK
  85
  86 #undef CALC_COUL_TAB
  87
  88 /* Analytical Ewald exclusion interaction electrostatics kernels */
  89 #define CALC_COUL_EWALD
  90
  91 /* Single cut-off: rcoulomb = rvdw */
  92 #include "nbnxn_kernel_simd_2xnn_includes.h"
  93
  94 /* Twin cut-off: rcoulomb >= rvdw */
  95 #define VDW_CUTOFF_CHECK
  96 #include "nbnxn_kernel_simd_2xnn_includes.h"
  97 #undef VDW_CUTOFF_CHECK
  98
  99 #undef CALC_COUL_EWALD
 100
 101
 102 typedef void (*p_nbk_func_ener)(const nbnxn_pairlist_t     *nbl,
 103                                 const nbnxn_atomdata_t     *nbat,
 104                                 const interaction_const_t  *ic,
 105                                 rvec                       *shift_vec,
 106                                 real                       *f,
 107                                 real                       *fshift,
 108                                 real                       *Vvdw,
 109                                 real                       *Vc);
 110
 111 typedef void (*p_nbk_func_noener)(const nbnxn_pairlist_t     *nbl,
 112                                   const nbnxn_atomdata_t     *nbat,
 113                                   const interaction_const_t  *ic,
 114                                   rvec                       *shift_vec,
 115                                   real                       *f,
 116                                   real                       *fshift);
 117
 118 enum { coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR };
 119
 120 #define NBK_FN(elec,ljcomb) nbnxn_kernel_simd_2xnn_##elec##_comb_##ljcomb##_ener
 121 static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
 122 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 123   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 124   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 125   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 126   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 127 #undef NBK_FN
 128
 129 #define NBK_FN(elec,ljcomb) nbnxn_kernel_simd_2xnn_##elec##_comb_##ljcomb##_energrp
 130 static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
 131 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 132   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 133   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 134   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 135   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 136 #undef NBK_FN
 137
 138 #define NBK_FN(elec,ljcomb) nbnxn_kernel_simd_2xnn_##elec##_comb_##ljcomb##_noener
 139 static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
 140 { { NBK_FN(rf        ,geom), NBK_FN(rf        ,lb), NBK_FN(rf        ,none) },
 141   { NBK_FN(tab       ,geom), NBK_FN(tab       ,lb), NBK_FN(tab       ,none) },
 142   { NBK_FN(tab_twin  ,geom), NBK_FN(tab_twin  ,lb), NBK_FN(tab_twin  ,none) },
 143   { NBK_FN(ewald     ,geom), NBK_FN(ewald     ,lb), NBK_FN(ewald     ,none) },
 144   { NBK_FN(ewald_twin,geom), NBK_FN(ewald_twin,lb), NBK_FN(ewald_twin,none) } };
 145 #undef NBK_FN
 146
 147
 148 static void reduce_group_energies(int ng,int ng_2log,
 149                                   const real *VSvdw,const real *VSc,
 150                                   real *Vvdw,real *Vc)
 151 {
 152     const int simd_width   = GMX_SIMD_WIDTH_HERE;
 153     const int unrollj_half = GMX_SIMD_WIDTH_HERE/4;
 154     int ng_p2,i,j,j0,j1,c,s;
 155
 156     ng_p2 = (1<<ng_2log);
 157
 158     /* The size of the x86 SIMD energy group buffer array is:
 159      * ng*ng*ng_p2*unrollj_half*simd_width
 160      */
 161     for(i=0; i<ng; i++)
 162     {
 163         for(j=0; j<ng; j++)
 164         {
 165             Vvdw[i*ng+j] = 0;
 166             Vc[i*ng+j]   = 0;
 167         }
 168
 169         for(j1=0; j1<ng; j1++)
 170         {
 171             for(j0=0; j0<ng; j0++)
 172             {
 173                 c = ((i*ng + j1)*ng_p2 + j0)*unrollj_half*simd_width/2;
 174                 for(s=0; s<unrollj_half; s++)
 175                 {
 176                     Vvdw[i*ng+j0] += VSvdw[c+0];
 177                     Vvdw[i*ng+j1] += VSvdw[c+1];
 178                     Vc  [i*ng+j0] += VSc  [c+0];
 179                     Vc  [i*ng+j1] += VSc  [c+1];
 180                     c += simd_width/2 + 2;
 181                 }
 182             }
 183         }
 184     }
 185 }
 186
 187 #endif /* GMX_NBNXN_SIMD_2XNN */
 188
 189 void
 190 nbnxn_kernel_simd_2xnn(nbnxn_pairlist_set_t       *nbl_list,
 191                        const nbnxn_atomdata_t     *nbat,
 192                        const interaction_const_t  *ic,
 193                        int                        ewald_excl,
 194                        rvec                       *shift_vec,
 195                        int                        force_flags,
 196                        int                        clearF,
 197                        real                       *fshift,
 198                        real                       *Vc,
 199                        real                       *Vvdw)
 200 #ifdef GMX_NBNXN_SIMD_2XNN
 201 {
 202     int              nnbl;
 203     nbnxn_pairlist_t **nbl;
 204     int coult;
 205     int nb;
 206
 207     nnbl = nbl_list->nnbl;
 208     nbl  = nbl_list->nbl;
 209
 210     if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
 211     {
 212         coult = coultRF;
 213     }
 214     else
 215     {
 216         if (ewald_excl == ewaldexclTable)
 217         {
 218             if (ic->rcoulomb == ic->rvdw)
 219             {
 220                 coult = coultTAB;
 221             }
 222             else
 223             {
 224                 coult = coultTAB_TWIN;
 225             }
 226         }
 227         else
 228         {
 229             if (ic->rcoulomb == ic->rvdw)
 230             {
 231                 coult = coultEWALD;
 232             }
 233             else
 234             {
 235                 coult = coultEWALD_TWIN;
 236             }
 237         }
 238     }
 239
 240 #pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
 241     for(nb=0; nb<nnbl; nb++)
 242     {
 243         nbnxn_atomdata_output_t *out;
 244         real *fshift_p;
 245
 246         out = &nbat->out[nb];
 247
 248         if (clearF == enbvClearFYes)
 249         {
 250             clear_f(nbat,nb,out->f);
 251         }
 252
 253         if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
 254         {
 255             fshift_p = fshift;
 256         }
 257         else
 258         {
 259             fshift_p = out->fshift;
 260
 261             if (clearF == enbvClearFYes)
 262             {
 263                 clear_fshift(fshift_p);
 264             }
 265         }
 266
 267         /* With Ewald type electrostatics we the forces for excluded atom pairs
 268          * should not contribute to the virial sum. The exclusion forces
 269          * are not calculate in the energy kernels, but are in _noener.
 270          */
 271         if (!((force_flags & GMX_FORCE_ENERGY) ||
 272               (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
 273         {
 274             /* Don't calculate energies */
 275             p_nbk_noener[coult][nbat->comb_rule](nbl[nb],nbat,
 276                                                  ic,
 277                                                  shift_vec,
 278                                                  out->f,
 279                                                  fshift_p);
 280         }
 281         else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
 282         {
 283             /* No energy groups */
 284             out->Vvdw[0] = 0;
 285             out->Vc[0]   = 0;
 286
 287             p_nbk_ener[coult][nbat->comb_rule](nbl[nb],nbat,
 288                                                ic,
 289                                                shift_vec,
 290                                                out->f,
 291                                                fshift_p,
 292                                                out->Vvdw,
 293                                                out->Vc);
 294         }
 295         else
 296         {
 297             /* Calculate energy group contributions */
 298             int i;
 299
 300             for(i=0; i<out->nVS; i++)
 301             {
 302                 out->VSvdw[i] = 0;
 303             }
 304             for(i=0; i<out->nVS; i++)
 305             {
 306                 out->VSc[i] = 0;
 307             }
 308
 309             p_nbk_energrp[coult][nbat->comb_rule](nbl[nb],nbat,
 310                                                   ic,
 311                                                   shift_vec,
 312                                                   out->f,
 313                                                   fshift_p,
 314                                                   out->VSvdw,
 315                                                   out->VSc);
 316
 317             reduce_group_energies(nbat->nenergrp,nbat->neg_2log,
 318                                   out->VSvdw,out->VSc,
 319                                   out->Vvdw,out->Vc);
 320         }
 321     }
 322
 323     if (force_flags & GMX_FORCE_ENERGY)
 324     {
 325         reduce_energies_over_lists(nbat,nnbl,Vvdw,Vc);
 326     }
 327 }
 328 #else
 329 {
 330     gmx_incons("nbnxn_kernel_simd_2xnn called while GROMACS was configured without 2x(N+N) SIMD kernels enabled");
 331 }
 332 #endif