2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team.
6 * Copyright (c) 2013,2014,2015,2016,2017,2018, by the GROMACS development team, led by
7 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
8 * and including many others, as listed in the AUTHORS file in the
9 * top-level source directory and at http://www.gromacs.org.
11 * GROMACS is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * GROMACS is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with GROMACS; if not, see
23 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
24 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 * If you want to redistribute modifications to GROMACS, please
27 * consider that scientific software is very special. Version
28 * control is crucial - bugs must be traceable. We will be happy to
29 * consider code for inclusion in the official distribution, but
30 * derived work must not be called official GROMACS. Details are found
31 * in the README & COPYING files - if they are missing, get the
32 * official version at http://www.gromacs.org.
34 * To help us fund GROMACS development, we humbly ask that you cite
35 * the research papers on the package. Check out http://www.gromacs.org.
38 * \brief This file defines functions for managing threading of listed
41 * \author Mark Abraham <mark.j.abraham@gmail.com>
42 * \ingroup module_listed-forces
46 #include "manage-threading.h"
57 #include "gromacs/listed-forces/listed-forces.h"
58 #include "gromacs/mdlib/gmx_omp_nthreads.h"
59 #include "gromacs/pbcutil/ishift.h"
60 #include "gromacs/topology/ifunc.h"
61 #include "gromacs/utility/exceptions.h"
62 #include "gromacs/utility/fatalerror.h"
63 #include "gromacs/utility/smalloc.h"
64 #include "gromacs/utility/stringutil.h"
66 #include "listed-internal.h"
68 /*! \brief struct for passing all data required for a function type */
70 const t_ilist *il; /**< pointer to t_ilist entry corresponding to ftype */
71 int ftype; /**< the function type index */
72 int nat; /**< nr of atoms involved in a single ftype interaction */
75 /*! \brief Divides listed interactions over threads
77 * This routine attempts to divide all interactions of the ntype bondeds
78 * types stored in ild over the threads such that each thread has roughly
79 * equal load and different threads avoid touching the same atoms as much
82 static void divide_bondeds_by_locality(bonded_threading_t *bt,
84 const ilist_data_t *ild)
87 int ind[F_NRE]; /* index into the ild[].il->iatoms */
88 int at_ind[F_NRE]; /* index of the first atom of the interaction at ind */
91 assert(ntype <= F_NRE);
94 for (f = 0; f < ntype; f++)
96 /* Sum #bondeds*#atoms_per_bond over all bonded types */
97 nat_tot += ild[f].il->nr/(ild[f].nat + 1)*ild[f].nat;
98 /* The start bound for thread 0 is 0 for all interactions */
100 /* Initialize the next atom index array */
101 assert(ild[f].il->nr > 0);
102 at_ind[f] = ild[f].il->iatoms[1];
106 /* Loop over the end bounds of the nthreads threads to determine
107 * which interactions threads 0 to nthreads shall calculate.
109 * NOTE: The cost of these combined loops is #interactions*ntype.
110 * This code is running single threaded (difficult to parallelize
111 * over threads). So the relative cost of this function increases
112 * linearly with the number of threads. Since the inner-most loop
113 * is cheap and this is done only at DD repartitioning, the cost should
114 * be negligble. At high thread count many other parts of the code
115 * scale the same way, so it's (currently) not worth improving this.
117 for (t = 1; t <= bt->nthreads; t++)
121 /* Here we assume that the computational cost is proportional
122 * to the number of atoms in the interaction. This is a rough
123 * measure, but roughly correct. Usually there are very few
124 * interactions anyhow and there are distributed relatively
125 * uniformly. Proper and RB dihedrals are often distributed
126 * non-uniformly, but their cost is roughly equal.
128 nat_thread = (nat_tot*t)/bt->nthreads;
130 while (nat_sum < nat_thread)
132 /* To divide bonds based on atom order, we compare
133 * the index of the first atom in the bonded interaction.
134 * This works well, since the domain decomposition generates
135 * bondeds in order of the atoms by looking up interactions
136 * which are linked to the first atom in each interaction.
137 * It usually also works well without DD, since than the atoms
138 * in bonded interactions are usually in increasing order.
139 * If they are not assigned in increasing order, the balancing
140 * is still good, but the memory access and reduction cost will
145 /* Find out which of the types has the lowest atom index */
147 for (f = 1; f < ntype; f++)
149 if (at_ind[f] < at_ind[f_min])
154 assert(f_min >= 0 && f_min < ntype);
156 /* Assign the interaction with the lowest atom index (of type
157 * index f_min) to thread t-1 by increasing ind.
159 ind[f_min] += ild[f_min].nat + 1;
160 nat_sum += ild[f_min].nat;
162 /* Update the first unassigned atom index for this type */
163 if (ind[f_min] < ild[f_min].il->nr)
165 at_ind[f_min] = ild[f_min].il->iatoms[ind[f_min] + 1];
169 /* We have assigned all interactions of this type.
170 * Setting at_ind to INT_MAX ensures this type will not be
171 * chosen in the for loop above during next iterations.
173 at_ind[f_min] = INT_MAX;
177 /* Store the bonded end boundaries (at index t) for thread t-1 */
178 for (f = 0; f < ntype; f++)
180 bt->il_thread_division[ild[f].ftype*(bt->nthreads + 1) + t] = ind[f];
184 for (f = 0; f < ntype; f++)
186 assert(ind[f] == ild[f].il->nr);
190 //! Divides bonded interactions over threads
191 static void divide_bondeds_over_threads(bonded_threading_t *bt,
194 ilist_data_t ild[F_NRE];
198 assert(bt->nthreads > 0);
200 if (F_NRE*(bt->nthreads + 1) > bt->il_thread_division_nalloc)
202 bt->il_thread_division_nalloc = F_NRE*(bt->nthreads + 1);
203 srenew(bt->il_thread_division, bt->il_thread_division_nalloc);
206 bt->haveBondeds = false;
208 for (f = 0; f < F_NRE; f++)
210 if (!ftype_is_bonded_potential(f))
215 if (idef->il[f].nr > 0)
217 bt->haveBondeds = true;
220 if (idef->il[f].nr == 0)
222 /* No interactions, avoid all the integer math below */
224 for (t = 0; t <= bt->nthreads; t++)
226 bt->il_thread_division[f*(bt->nthreads + 1) + t] = 0;
229 else if (bt->nthreads <= bt->max_nthread_uniform || f == F_DISRES)
231 /* On up to 4 threads, load balancing the bonded work
232 * is more important than minimizing the reduction cost.
236 /* nat1 = 1 + #atoms(ftype) which is the stride use for iatoms */
239 for (t = 0; t <= bt->nthreads; t++)
243 /* Divide equally over the threads */
244 nr_t = (((idef->il[f].nr/nat1)*t)/bt->nthreads)*nat1;
248 /* Ensure that distance restraint pairs with the same label
249 * end up on the same thread.
251 while (nr_t > 0 && nr_t < idef->il[f].nr &&
252 idef->iparams[idef->il[f].iatoms[nr_t]].disres.label ==
253 idef->iparams[idef->il[f].iatoms[nr_t-nat1]].disres.label)
259 bt->il_thread_division[f*(bt->nthreads + 1) + t] = nr_t;
264 /* Add this ftype to the list to be distributed */
268 ild[ntype].ftype = f;
269 ild[ntype].il = &idef->il[f];
270 ild[ntype].nat = nat;
272 /* The first index for the thread division is always 0 */
273 bt->il_thread_division[f*(bt->nthreads + 1)] = 0;
281 divide_bondeds_by_locality(bt, ntype, ild);
288 fprintf(debug, "Division of bondeds over threads:\n");
289 for (f = 0; f < F_NRE; f++)
291 if (ftype_is_bonded_potential(f) && idef->il[f].nr > 0)
295 fprintf(debug, "%16s", interaction_function[f].name);
296 for (t = 0; t < bt->nthreads; t++)
298 fprintf(debug, " %4d",
299 (bt->il_thread_division[f*(bt->nthreads + 1) + t + 1] -
300 bt->il_thread_division[f*(bt->nthreads + 1) + t])/
303 fprintf(debug, "\n");
309 //! Construct a reduction mask for which parts (blocks) of the force array are touched on which thread task
311 calc_bonded_reduction_mask(int natoms,
312 f_thread_t *f_thread,
315 const bonded_threading_t &bondedThreading)
317 static_assert(BITMASK_SIZE == GMX_OPENMP_MAX_THREADS, "For the error message below we assume these two are equal.");
319 if (bondedThreading.nthreads > BITMASK_SIZE)
322 gmx_fatal(FARGS, "You are using %d OpenMP threads, which is larger than GMX_OPENMP_MAX_THREADS (%d). Decrease the number of OpenMP threads or rebuild GROMACS with a larger value for GMX_OPENMP_MAX_THREADS.",
323 bondedThreading.nthreads, GMX_OPENMP_MAX_THREADS);
326 GMX_ASSERT(bondedThreading.nthreads <= BITMASK_SIZE, "We need at least nthreads bits in the mask");
328 int nblock = (natoms + reduction_block_size - 1) >> reduction_block_bits;
330 if (nblock > f_thread->block_nalloc)
332 f_thread->block_nalloc = over_alloc_large(nblock);
333 srenew(f_thread->mask, f_thread->block_nalloc);
334 srenew(f_thread->block_index, f_thread->block_nalloc);
335 sfree_aligned(f_thread->f);
336 snew_aligned(f_thread->f, f_thread->block_nalloc*reduction_block_size, 128);
339 gmx_bitmask_t *mask = f_thread->mask;
341 for (int b = 0; b < nblock; b++)
343 bitmask_clear(&mask[b]);
346 for (int ftype = 0; ftype < F_NRE; ftype++)
348 if (ftype_is_bonded_potential(ftype))
350 int nb = idef->il[ftype].nr;
353 int nat1 = interaction_function[ftype].nratoms + 1;
355 int nb0 = bondedThreading.il_thread_division[ftype*(bondedThreading.nthreads + 1) + thread];
356 int nb1 = bondedThreading.il_thread_division[ftype*(bondedThreading.nthreads + 1) + thread + 1];
358 for (int i = nb0; i < nb1; i += nat1)
360 for (int a = 1; a < nat1; a++)
362 bitmask_set_bit(&mask[idef->il[ftype].iatoms[i+a] >> reduction_block_bits], thread);
369 /* Make an index of the blocks our thread touches, so we can do fast
370 * force buffer clearing.
372 f_thread->nblock_used = 0;
373 for (int b = 0; b < nblock; b++)
375 if (bitmask_is_set(mask[b], thread))
377 f_thread->block_index[f_thread->nblock_used++] = b;
382 void setup_bonded_threading(bonded_threading_t *bt,
388 assert(bt->nthreads >= 1);
390 /* Divide the bonded interaction over the threads */
391 divide_bondeds_over_threads(bt, idef);
393 if (!bt->haveBondeds)
395 /* We don't have bondeds, so there is nothing to reduce */
399 /* Determine to which blocks each thread's bonded force calculation
400 * contributes. Store this as a mask for each thread.
402 #pragma omp parallel for num_threads(bt->nthreads) schedule(static)
403 for (int t = 0; t < bt->nthreads; t++)
407 calc_bonded_reduction_mask(numAtoms, &bt->f_t[t],
410 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
413 /* Reduce the masks over the threads and determine which blocks
414 * we need to reduce over.
416 int nblock_tot = (numAtoms + reduction_block_size - 1) >> reduction_block_bits;
417 if (nblock_tot > bt->block_nalloc)
419 bt->block_nalloc = over_alloc_large(nblock_tot);
420 srenew(bt->block_index, bt->block_nalloc);
421 srenew(bt->mask, bt->block_nalloc);
424 for (int b = 0; b < nblock_tot; b++)
426 gmx_bitmask_t *mask = &bt->mask[b];
428 /* Generate the union over the threads of the bitmask */
430 for (int t = 0; t < bt->nthreads; t++)
432 bitmask_union(mask, bt->f_t[t].mask[b]);
434 if (!bitmask_is_zero(*mask))
436 bt->block_index[bt->nblock_used++] = b;
442 for (int t = 0; t < bt->nthreads; t++)
444 if (bitmask_is_set(*mask, t))
453 #if BITMASK_SIZE <= 64 //move into bitmask when it is C++
454 std::string flags = gmx::formatString("%" PRIx64, *mask);
456 std::string flags = gmx::formatAndJoin(*mask,
457 "", gmx::StringFormatter("%x"));
459 fprintf(debug, "block %d flags %s count %d\n",
460 b, flags.c_str(), c);
466 fprintf(debug, "Number of %d atom blocks to reduce: %d\n",
467 reduction_block_size, bt->nblock_used);
468 fprintf(debug, "Reduction density %.2f for touched blocks only %.2f\n",
469 ctot*reduction_block_size/static_cast<double>(numAtoms),
470 ctot/static_cast<double>(bt->nblock_used));
474 void tear_down_bonded_threading(bonded_threading_t *bt)
476 for (int th = 0; th < bt->nthreads; th++)
478 sfree(bt->f_t[th].fshift);
479 for (int i = 0; i < egNR; i++)
481 sfree(bt->f_t[th].grpp.ener[i]);
485 sfree(bt->il_thread_division);
489 void init_bonded_threading(FILE *fplog, int nenergrp,
490 struct bonded_threading_t **bt_ptr)
492 bonded_threading_t *bt;
496 /* These thread local data structures are used for bondeds only.
498 * Note that we also use there structures when running single-threaded.
499 * This is because the bonded force buffer uses type rvec4, whereas
500 * the normal force buffer is uses type rvec. This leads to a little
501 * reduction overhead, but the speed gain in the bonded calculations
502 * of doing transposeScatterIncr/DecrU with aligment 4 instead of 3
503 * is much larger than the reduction overhead.
505 bt->nthreads = gmx_omp_nthreads_get(emntBonded);
507 snew(bt->f_t, bt->nthreads);
508 #pragma omp parallel for num_threads(bt->nthreads) schedule(static)
509 for (int t = 0; t < bt->nthreads; t++)
513 /* Note that thread 0 uses the global fshift and energy arrays,
514 * but to keep the code simple, we initialize all data here.
516 bt->f_t[t].f = nullptr;
517 bt->f_t[t].f_nalloc = 0;
518 snew(bt->f_t[t].fshift, SHIFTS);
519 bt->f_t[t].grpp.nener = nenergrp*nenergrp;
520 for (int i = 0; i < egNR; i++)
522 snew(bt->f_t[t].grpp.ener[i], bt->f_t[t].grpp.nener);
525 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
529 bt->block_index = nullptr;
531 bt->block_nalloc = 0;
533 /* The optimal value after which to switch from uniform to localized
534 * bonded interaction distribution is 3, 4 or 5 depending on the system
537 const int max_nthread_uniform = 4;
540 if ((ptr = getenv("GMX_BONDED_NTHREAD_UNIFORM")) != nullptr)
542 sscanf(ptr, "%d", &bt->max_nthread_uniform);
543 if (fplog != nullptr)
545 fprintf(fplog, "\nMax threads for uniform bonded distribution set to %d by env.var.\n",
546 bt->max_nthread_uniform);
551 bt->max_nthread_uniform = max_nthread_uniform;