src/mdlib/nbnxn_kernels/nbnxn_kernel_simd_2xnn.c

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2012, The GROMACS development team,
   6  * check out http://www.gromacs.org for more information.
   7  * Copyright (c) 2012,2013, by the GROMACS development team, led by
   8  * David van der Spoel, Berk Hess, Erik Lindahl, and including many
   9  * others, as listed in the AUTHORS file in the top-level source
  10  * directory and at http://www.gromacs.org.
  11  *
  12  * GROMACS is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU Lesser General Public License
  14  * as published by the Free Software Foundation; either version 2.1
  15  * of the License, or (at your option) any later version.
  16  *
  17  * GROMACS is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20  * Lesser General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public
  23  * License along with GROMACS; if not, see
  24  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  25  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  26  *
  27  * If you want to redistribute modifications to GROMACS, please
  28  * consider that scientific software is very special. Version
  29  * control is crucial - bugs must be traceable. We will be happy to
  30  * consider code for inclusion in the official distribution, but
  31  * derived work must not be called official GROMACS. Details are found
  32  * in the README & COPYING files - if they are missing, get the
  33  * official version at http://www.gromacs.org.
  34  *
  35  * To help us fund GROMACS development, we humbly ask that you cite
  36  * the research papers on the package. Check out http://www.gromacs.org.
  37  */
  38 #ifdef HAVE_CONFIG_H
  39 #include <config.h>
  40 #endif
  41
  42 #include <math.h>
  43
  44 #include "typedefs.h"
  45 #include "vec.h"
  46 #include "smalloc.h"
  47 #include "force.h"
  48 #include "gmx_omp_nthreads.h"
  49 #include "../nbnxn_consts.h"
  50 #include "nbnxn_kernel_common.h"
  51
  52 #ifdef GMX_NBNXN_SIMD_2XNN
  53
  54 /* Include the full width SIMD macros */
  55 #include "gmx_simd_macros.h"
  56 #include "gmx_simd_vec.h"
  57
  58 #include "nbnxn_kernel_simd_2xnn.h"
  59
  60 #if !(GMX_SIMD_WIDTH_HERE == 8 || GMX_SIMD_WIDTH_HERE == 16)
  61 #error "unsupported SIMD width"
  62 #endif
  63
  64
  65 /* Include all flavors of the SSE or AVX 2x(N+N) kernel loops */
  66
  67 /* Analytical reaction-field kernels */
  68 #define CALC_COUL_RF
  69
  70 #include "nbnxn_kernel_simd_2xnn_includes.h"
  71
  72 #undef CALC_COUL_RF
  73
  74 /* Tabulated exclusion interaction electrostatics kernels */
  75 #define CALC_COUL_TAB
  76
  77 /* Single cut-off: rcoulomb = rvdw */
  78 #include "nbnxn_kernel_simd_2xnn_includes.h"
  79
  80 /* Twin cut-off: rcoulomb >= rvdw */
  81 #define VDW_CUTOFF_CHECK
  82 #include "nbnxn_kernel_simd_2xnn_includes.h"
  83 #undef VDW_CUTOFF_CHECK
  84
  85 #undef CALC_COUL_TAB
  86
  87 /* Analytical Ewald exclusion interaction electrostatics kernels */
  88 #define CALC_COUL_EWALD
  89
  90 /* Single cut-off: rcoulomb = rvdw */
  91 #include "nbnxn_kernel_simd_2xnn_includes.h"
  92
  93 /* Twin cut-off: rcoulomb >= rvdw */
  94 #define VDW_CUTOFF_CHECK
  95 #include "nbnxn_kernel_simd_2xnn_includes.h"
  96 #undef VDW_CUTOFF_CHECK
  97
  98 #undef CALC_COUL_EWALD
  99
 100
 101 typedef void (*p_nbk_func_ener)(const nbnxn_pairlist_t     *nbl,
 102                                 const nbnxn_atomdata_t     *nbat,
 103                                 const interaction_const_t  *ic,
 104                                 rvec                       *shift_vec,
 105                                 real                       *f,
 106                                 real                       *fshift,
 107                                 real                       *Vvdw,
 108                                 real                       *Vc);
 109
 110 typedef void (*p_nbk_func_noener)(const nbnxn_pairlist_t     *nbl,
 111                                   const nbnxn_atomdata_t     *nbat,
 112                                   const interaction_const_t  *ic,
 113                                   rvec                       *shift_vec,
 114                                   real                       *f,
 115                                   real                       *fshift);
 116
 117 enum {
 118     coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR
 119 };
 120
 121 #define NBK_FN(elec, ljcomb) nbnxn_kernel_simd_2xnn_ ## elec ## _comb_ ## ljcomb ## _ener
 122 static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
 123 { { NBK_FN(rf, geom), NBK_FN(rf, lb), NBK_FN(rf, none) },
 124   { NBK_FN(tab, geom), NBK_FN(tab, lb), NBK_FN(tab, none) },
 125   { NBK_FN(tab_twin, geom), NBK_FN(tab_twin, lb), NBK_FN(tab_twin, none) },
 126   { NBK_FN(ewald, geom), NBK_FN(ewald, lb), NBK_FN(ewald, none) },
 127   { NBK_FN(ewald_twin, geom), NBK_FN(ewald_twin, lb), NBK_FN(ewald_twin, none) } };
 128 #undef NBK_FN
 129
 130 #define NBK_FN(elec, ljcomb) nbnxn_kernel_simd_2xnn_ ## elec ## _comb_ ## ljcomb ## _energrp
 131 static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
 132 { { NBK_FN(rf, geom), NBK_FN(rf, lb), NBK_FN(rf, none) },
 133   { NBK_FN(tab, geom), NBK_FN(tab, lb), NBK_FN(tab, none) },
 134   { NBK_FN(tab_twin, geom), NBK_FN(tab_twin, lb), NBK_FN(tab_twin, none) },
 135   { NBK_FN(ewald, geom), NBK_FN(ewald, lb), NBK_FN(ewald, none) },
 136   { NBK_FN(ewald_twin, geom), NBK_FN(ewald_twin, lb), NBK_FN(ewald_twin, none) } };
 137 #undef NBK_FN
 138
 139 #define NBK_FN(elec, ljcomb) nbnxn_kernel_simd_2xnn_ ## elec ## _comb_ ## ljcomb ## _noener
 140 static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
 141 { { NBK_FN(rf, geom), NBK_FN(rf, lb), NBK_FN(rf, none) },
 142   { NBK_FN(tab, geom), NBK_FN(tab, lb), NBK_FN(tab, none) },
 143   { NBK_FN(tab_twin, geom), NBK_FN(tab_twin, lb), NBK_FN(tab_twin, none) },
 144   { NBK_FN(ewald, geom), NBK_FN(ewald, lb), NBK_FN(ewald, none) },
 145   { NBK_FN(ewald_twin, geom), NBK_FN(ewald_twin, lb), NBK_FN(ewald_twin, none) } };
 146 #undef NBK_FN
 147
 148
 149 static void reduce_group_energies(int ng, int ng_2log,
 150                                   const real *VSvdw, const real *VSc,
 151                                   real *Vvdw, real *Vc)
 152 {
 153     const int unrollj      = GMX_SIMD_WIDTH_HERE/2;
 154     const int unrollj_half = unrollj/2;
 155     int       ng_p2, i, j, j0, j1, c, s;
 156
 157     ng_p2 = (1<<ng_2log);
 158
 159     /* The size of the x86 SIMD energy group buffer array is:
 160      * ng*ng*ng_p2*unrollj_half*simd_width
 161      */
 162     for (i = 0; i < ng; i++)
 163     {
 164         for (j = 0; j < ng; j++)
 165         {
 166             Vvdw[i*ng+j] = 0;
 167             Vc[i*ng+j]   = 0;
 168         }
 169
 170         for (j1 = 0; j1 < ng; j1++)
 171         {
 172             for (j0 = 0; j0 < ng; j0++)
 173             {
 174                 c = ((i*ng + j1)*ng_p2 + j0)*unrollj_half*unrollj;
 175                 for (s = 0; s < unrollj_half; s++)
 176                 {
 177                     Vvdw[i*ng+j0] += VSvdw[c+0];
 178                     Vvdw[i*ng+j1] += VSvdw[c+1];
 179                     Vc  [i*ng+j0] += VSc  [c+0];
 180                     Vc  [i*ng+j1] += VSc  [c+1];
 181                     c             += unrollj + 2;
 182                 }
 183             }
 184         }
 185     }
 186 }
 187
 188 #endif /* GMX_NBNXN_SIMD_2XNN */
 189
 190 void
 191 nbnxn_kernel_simd_2xnn(nbnxn_pairlist_set_t       *nbl_list,
 192                        const nbnxn_atomdata_t     *nbat,
 193                        const interaction_const_t  *ic,
 194                        int                         ewald_excl,
 195                        rvec                       *shift_vec,
 196                        int                         force_flags,
 197                        int                         clearF,
 198                        real                       *fshift,
 199                        real                       *Vc,
 200                        real                       *Vvdw)
 201 #ifdef GMX_NBNXN_SIMD_2XNN
 202 {
 203     int                nnbl;
 204     nbnxn_pairlist_t **nbl;
 205     int                coult;
 206     int                nb;
 207
 208     nnbl = nbl_list->nnbl;
 209     nbl  = nbl_list->nbl;
 210
 211     if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
 212     {
 213         coult = coultRF;
 214     }
 215     else
 216     {
 217         if (ewald_excl == ewaldexclTable)
 218         {
 219             if (ic->rcoulomb == ic->rvdw)
 220             {
 221                 coult = coultTAB;
 222             }
 223             else
 224             {
 225                 coult = coultTAB_TWIN;
 226             }
 227         }
 228         else
 229         {
 230             if (ic->rcoulomb == ic->rvdw)
 231             {
 232                 coult = coultEWALD;
 233             }
 234             else
 235             {
 236                 coult = coultEWALD_TWIN;
 237             }
 238         }
 239     }
 240
 241 #pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
 242     for (nb = 0; nb < nnbl; nb++)
 243     {
 244         nbnxn_atomdata_output_t *out;
 245         real                    *fshift_p;
 246
 247         out = &nbat->out[nb];
 248
 249         if (clearF == enbvClearFYes)
 250         {
 251             clear_f(nbat, nb, out->f);
 252         }
 253
 254         if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
 255         {
 256             fshift_p = fshift;
 257         }
 258         else
 259         {
 260             fshift_p = out->fshift;
 261
 262             if (clearF == enbvClearFYes)
 263             {
 264                 clear_fshift(fshift_p);
 265             }
 266         }
 267
 268         /* With Ewald type electrostatics we the forces for excluded atom pairs
 269          * should not contribute to the virial sum. The exclusion forces
 270          * are not calculate in the energy kernels, but are in _noener.
 271          */
 272         if (!((force_flags & GMX_FORCE_ENERGY) ||
 273               (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
 274         {
 275             /* Don't calculate energies */
 276             p_nbk_noener[coult][nbat->comb_rule](nbl[nb], nbat,
 277                                                  ic,
 278                                                  shift_vec,
 279                                                  out->f,
 280                                                  fshift_p);
 281         }
 282         else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
 283         {
 284             /* No energy groups */
 285             out->Vvdw[0] = 0;
 286             out->Vc[0]   = 0;
 287
 288             p_nbk_ener[coult][nbat->comb_rule](nbl[nb], nbat,
 289                                                ic,
 290                                                shift_vec,
 291                                                out->f,
 292                                                fshift_p,
 293                                                out->Vvdw,
 294                                                out->Vc);
 295         }
 296         else
 297         {
 298             /* Calculate energy group contributions */
 299             int i;
 300
 301             for (i = 0; i < out->nVS; i++)
 302             {
 303                 out->VSvdw[i] = 0;
 304             }
 305             for (i = 0; i < out->nVS; i++)
 306             {
 307                 out->VSc[i] = 0;
 308             }
 309
 310             p_nbk_energrp[coult][nbat->comb_rule](nbl[nb], nbat,
 311                                                   ic,
 312                                                   shift_vec,
 313                                                   out->f,
 314                                                   fshift_p,
 315                                                   out->VSvdw,
 316                                                   out->VSc);
 317
 318             reduce_group_energies(nbat->nenergrp, nbat->neg_2log,
 319                                   out->VSvdw, out->VSc,
 320                                   out->Vvdw, out->Vc);
 321         }
 322     }
 323
 324     if (force_flags & GMX_FORCE_ENERGY)
 325     {
 326         reduce_energies_over_lists(nbat, nnbl, Vvdw, Vc);
 327     }
 328 }
 329 #else
 330 {
 331     gmx_incons("nbnxn_kernel_simd_2xnn called while GROMACS was configured without 2x(N+N) SIMD kernels enabled");
 332 }
 333 #endif