static void dd_sort_order_nbnxn(const t_forcerec *fr,
std::vector<gmx_cgsort_t> *sort)
{
- gmx::ArrayRef<const int> atomOrder = nbnxn_get_atomorder(fr->nbv->nbs.get());
+ gmx::ArrayRef<const int> atomOrder = fr->nbv->getLocalAtomOrder();
/* Using push_back() instead of this resize results in much slower code */
sort->resize(atomOrder.size());
wallcycle_sub_stop(wcycle, ewcsNBS_GRID_NONLOCAL);
}
- nbnxn_atomdata_set(nbv->nbat.get(), nbv->nbs.get(), mdatoms, fr->cginfo);
+ nbv->setAtomProperties(*mdatoms, *fr->cginfo);
wallcycle_stop(wcycle, ewcNS);
// TODO the xq, f, and fshift buffers are now shared
// resources, so they should be maintained by a
// higher-level object than the nb module.
- fr->gpuBonded->updateInteractionListsAndDeviceBuffers(nbnxn_get_gridindices(fr->nbv->nbs.get()),
+ fr->gpuBonded->updateInteractionListsAndDeviceBuffers(nbv->getGridIndices(),
top->idef,
Nbnxm::gpu_get_xq(nbv->gpu_nbv),
Nbnxm::gpu_get_f(nbv->gpu_nbv),
Nbnxm::gpu_get_fshift(nbv->gpu_nbv));
}
-
wallcycle_stop(wcycle, ewcLAUNCH_GPU);
}
}
else
{
- nbnxn_atomdata_copy_x_to_nbat_x(nbv->nbs.get(), Nbnxm::AtomLocality::Local,
- FALSE, as_rvec_array(x.unpaddedArrayRef().data()),
- nbv->nbat.get(), wcycle);
+ nbv->setCoordinates(Nbnxm::AtomLocality::Local, false,
+ x.unpaddedArrayRef(), wcycle);
}
if (bUseGPU)
{
dd_move_x(cr->dd, box, x.unpaddedArrayRef(), wcycle);
- nbnxn_atomdata_copy_x_to_nbat_x(nbv->nbs.get(), Nbnxm::AtomLocality::NonLocal,
- FALSE, as_rvec_array(x.unpaddedArrayRef().data()),
- nbv->nbat.get(), wcycle);
+ nbv->setCoordinates(Nbnxm::AtomLocality::NonLocal, false,
+ x.unpaddedArrayRef(), wcycle);
}
if (bUseGPU)
/* Sets all required atom parameter data in nbnxn_atomdata_t */
void nbnxn_atomdata_set(nbnxn_atomdata_t *nbat,
- const nbnxn_search *nbs,
+ const PairSearch &pairSearch,
const t_mdatoms *mdatoms,
const int *atinfo)
{
nbnxn_atomdata_t::Params ¶ms = nbat->paramsDeprecated();
- const Nbnxm::GridSet &gridSet = nbs->gridSet();
+ const Nbnxm::GridSet &gridSet = pairSearch.gridSet();
nbnxn_atomdata_set_atomtypes(¶ms, gridSet, mdatoms->typeA);
}
/* Copies (and reorders) the coordinates to nbnxn_atomdata_t */
-void nbnxn_atomdata_copy_x_to_nbat_x(const nbnxn_search *nbs,
+void nbnxn_atomdata_copy_x_to_nbat_x(const PairSearch &pairSearch,
const Nbnxm::AtomLocality locality,
gmx_bool FillLocal,
- rvec *x,
+ const rvec *x,
nbnxn_atomdata_t *nbat,
gmx_wallcycle *wcycle)
{
wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
wallcycle_sub_start(wcycle, ewcsNB_X_BUF_OPS);
- const Nbnxm::GridSet &gridSet = nbs->gridSet();
+ const Nbnxm::GridSet &gridSet = pairSearch.gridSet();
int gridBegin = 0;
int gridEnd = 0;
wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
wallcycle_sub_start(wcycle, ewcsNB_F_BUF_OPS);
- const Nbnxm::GridSet &gridSet = nbs->gridSet();
+ const Nbnxm::GridSet &gridSet = pairSearch_->gridSet();
- nbs_cycle_start(&nbs->cc[enbsCCreducef]);
-
- int a0 = 0;
- int na = 0;
+ int a0 = 0;
+ int na = 0;
switch (locality)
{
case Nbnxm::AtomLocality::All:
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
}
- nbs_cycle_stop(&nbs->cc[enbsCCreducef]);
-
wallcycle_sub_stop(wcycle, ewcsNB_F_BUF_OPS);
wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
}
struct gmx_wallcycle;
struct nbnxn_atomdata_t;
-struct nbnxn_search;
struct nonbonded_verlet_t;
+class PairSearch;
struct t_mdatoms;
struct tMPI_Atomic;
int nout);
void nbnxn_atomdata_set(nbnxn_atomdata_t *nbat,
- const nbnxn_search *nbs,
+ const PairSearch &pairSearch,
const t_mdatoms *mdatoms,
const int *atinfo);
/* Copy x to nbat->x.
* FillLocal tells if the local filler particle coordinates should be zeroed.
*/
-void nbnxn_atomdata_copy_x_to_nbat_x(const nbnxn_search *nbs,
+void nbnxn_atomdata_copy_x_to_nbat_x(const PairSearch &pairSearch,
Nbnxm::AtomLocality locality,
gmx_bool FillLocal,
- rvec *x,
+ const rvec *x,
nbnxn_atomdata_t *nbat,
gmx_wallcycle *wcycle);
#include "gridset.h"
-#include "gromacs/domdec/domdec_struct.h"
#include "gromacs/mdlib/gmx_omp_nthreads.h"
#include "gromacs/mdlib/updategroupscog.h"
-#include "gromacs/nbnxm/nbnxm.h"
+#include "gromacs/nbnxm/atomdata.h"
#include "gromacs/utility/fatalerror.h"
-#include "internal.h"
-
namespace Nbnxm
{
}
} // namespace Nbnxm
-
-// TODO: Move this function to a proper location after refactoring nbnxn_search
-void nbnxn_put_on_grid(nonbonded_verlet_t *nb_verlet,
- const matrix box,
- int ddZone,
- const rvec lowerCorner,
- const rvec upperCorner,
- const gmx::UpdateGroupsCog *updateGroupsCog,
- int atomStart,
- int atomEnd,
- real atomDensity,
- const int *atinfo,
- gmx::ArrayRef<const gmx::RVec> x,
- int numAtomsMoved,
- const int *move)
-{
- nbnxn_search &nbs = *nb_verlet->nbs;
-
- nbs_cycle_start(&nbs.cc[enbsCCgrid]);
-
- nbs.gridSet_.putOnGrid(box, ddZone, lowerCorner, upperCorner,
- updateGroupsCog, atomStart, atomEnd, atomDensity,
- atinfo, x, numAtomsMoved, move,
- nb_verlet->nbat.get());
-
- nbs_cycle_stop(&nbs.cc[enbsCCgrid]);
-}
-
-/* Calls nbnxn_put_on_grid for all non-local domains */
-void nbnxn_put_on_grid_nonlocal(nonbonded_verlet_t *nbv,
- const struct gmx_domdec_zones_t *zones,
- const int *atinfo,
- gmx::ArrayRef<const gmx::RVec> x)
-{
- for (int zone = 1; zone < zones->n; zone++)
- {
- rvec c0, c1;
- for (int d = 0; d < DIM; d++)
- {
- c0[d] = zones->size[zone].bb_x0[d];
- c1[d] = zones->size[zone].bb_x1[d];
- }
-
- nbnxn_put_on_grid(nbv, nullptr,
- zone, c0, c1,
- nullptr,
- zones->cg_range[zone],
- zones->cg_range[zone+1],
- -1,
- atinfo,
- x,
- 0, nullptr);
- }
-}
-
-gmx::ArrayRef<const int> nbnxn_get_atomorder(const nbnxn_search *nbs)
-{
- /* Return the atom order for the home cell (index 0) */
- const Nbnxm::Grid &grid = nbs->gridSet().grids()[0];
-
- const int numIndices = grid.atomIndexEnd() - grid.firstAtomInColumn(0);
-
- return gmx::constArrayRefFromArray(nbs->gridSet().atomIndices().data(), numIndices);
-}
-
-void nonbonded_verlet_t::setLocalAtomOrder()
-{
- nbs->gridSet_.setLocalAtomOrder();
-}
-
-void nonbonded_verlet_t::getLocalNumCells(int *numCellsX,
- int *numCellsY) const
-{
- nbs->gridSet().getLocalNumCells(numCellsX, numCellsY);
-}
-
-gmx::ArrayRef<const int> nbnxn_get_gridindices(const nbnxn_search* nbs)
-{
- return nbs->gridSet().cells();
-}
/* Local cycle count struct for profiling */
-typedef struct {
- int count;
- gmx_cycles_t c;
- gmx_cycles_t start;
-} nbnxn_cycle_t;
-
-/* Local cycle count enum for profiling */
-enum {
- enbsCCgrid, enbsCCsearch, enbsCCcombine, enbsCCreducef, enbsCCnr
+class nbnxn_cycle_t
+{
+ public:
+ void start()
+ {
+ start_ = gmx_cycles_read();
+ }
+
+ void stop()
+ {
+ cycles_ += gmx_cycles_read() - start_;
+ count_++;
+ }
+
+ int count() const
+ {
+ return count_;
+ }
+
+ double averageMCycles() const
+ {
+ if (count_ > 0)
+ {
+ return static_cast<double>(cycles_)*1e-6/count_;
+ }
+ else
+ {
+ return 0;
+ }
+ }
+
+ private:
+ int count_ = 0;
+ gmx_cycles_t cycles_ = 0;
+ gmx_cycles_t start_ = 0;
};
+// TODO: Move nbnxn_search_work_t definition to its own file
+
/* Thread-local work struct, contains working data for Grid */
-struct nbnxn_search_work_t
+struct PairsearchWork
{
- nbnxn_search_work_t();
+ PairsearchWork();
- ~nbnxn_search_work_t();
+ ~PairsearchWork();
gmx_cache_protect_t cp0; /* Buffer to avoid cache polution */
std::unique_ptr<t_nblist> nbl_fep; /* Temporary FEP list for load balancing */
- nbnxn_cycle_t cc[enbsCCnr]; /* Counters for thread-local cycles */
+ nbnxn_cycle_t cycleCounter; /* Counter for thread-local cycles */
gmx_cache_protect_t cp1; /* Buffer to avoid cache polution */
};
/* Main pair-search struct, contains the grid(s), not the pair-list(s) */
-struct nbnxn_search
+class PairSearch
{
- /*! \internal
- * \brief Description of the domain setup: PBC and the connections between domains
- */
- struct DomainSetup
- {
- //! Constructor, without DD \p numDDCells and \p ddZones should be nullptr
- DomainSetup(int ePBC,
- const ivec *numDDCells,
- const gmx_domdec_zones_t *ddZones);
-
- //! The type of PBC
- int ePBC;
- //! Tells whether we are using domain decomposition
- bool haveDomDec;
- //! Tells whether we are using domain decomposition per dimension
- std::array<bool, DIM> haveDomDecPerDim;
- //! The domain decomposition zone setup
- const gmx_domdec_zones_t *zones;
- };
-
- /* \brief Constructor
- *
- * \param[in] ePBC The periodic boundary conditions
- * \param[in] numDDCells The number of domain decomposition cells per dimension, without DD nullptr should be passed
- * \param[in] zones The domain decomposition zone setup, without DD nullptr should be passed
- * \param[in] haveFep Tells whether non-bonded interactions are perturbed
- * \param[in] maxNumThreads The maximum number of threads used in the search
- */
- nbnxn_search(int ePBC,
- const ivec *numDDCells,
- const gmx_domdec_zones_t *zones,
- PairlistType pairlistType,
- bool haveFep,
- int maxNumthreads);
-
- const DomainSetup &domainSetup() const
- {
- return domainSetup_;
- }
-
- const Nbnxm::GridSet &gridSet() const
- {
- return gridSet_;
- }
-
- //! The domain setup
- DomainSetup domainSetup_;
-
- //! The local and non-local grids
- Nbnxm::GridSet gridSet_;
-
- gmx_bool print_cycles;
- int search_count;
- nbnxn_cycle_t cc[enbsCCnr];
-
- /* Thread-local work data */
- mutable std::vector<nbnxn_search_work_t> work; /* Work array, one entry for each thread */
+ public:
+ /*! \internal
+ * \brief Description of the domain setup: PBC and the connections between domains
+ */
+ struct DomainSetup
+ {
+ /*! \internal
+ * \brief Description of the domain setup: PBC and the connections between domains
+ */
+ //! Constructor, without DD \p numDDCells and \p ddZones should be nullptr
+ DomainSetup(int ePBC,
+ const ivec *numDDCells,
+ const gmx_domdec_zones_t *ddZones);
+
+ //! The type of PBC
+ int ePBC;
+ //! Tells whether we are using domain decomposition
+ bool haveDomDec;
+ //! Tells whether we are using domain decomposition per dimension
+ std::array<bool, DIM> haveDomDecPerDim;
+ //! The domain decomposition zone setup
+ const gmx_domdec_zones_t *zones;
+ };
+
+ //! Local cycle count enum for profiling different parts of search
+ enum {
+ enbsCCgrid, enbsCCsearch, enbsCCcombine, enbsCCnr
+ };
+
+ struct SearchCycleCounting
+ {
+ //! Start a pair search cycle counter
+ void start(const int enbsCC)
+ {
+ cc_[enbsCC].start();
+ }
+
+ //! Stop a pair search cycle counter
+ void stop(const int enbsCC)
+ {
+ cc_[enbsCC].stop();
+ }
+
+ //! Print the cycle counts to \p fp
+ void printCycles(FILE *fp,
+ gmx::ArrayRef<const PairsearchWork> work) const;
+
+ bool recordCycles_ = false;
+ int searchCount_ = 0;
+ nbnxn_cycle_t cc_[enbsCCnr];
+ };
+
+ //! Puts the atoms in \p ddZone on the grid and copies the coordinates to \p nbat
+ void putOnGrid(const matrix box,
+ int ddZone,
+ const rvec lowerCorner,
+ const rvec upperCorner,
+ const gmx::UpdateGroupsCog *updateGroupsCog,
+ int atomStart,
+ int atomEnd,
+ real atomDensity,
+ const int *atinfo,
+ gmx::ArrayRef<const gmx::RVec> x,
+ int numAtomsMoved,
+ const int *move,
+ nbnxn_atomdata_t *nbat)
+ {
+ cycleCounting_.start(enbsCCgrid);
+
+ gridSet_.putOnGrid(box, ddZone, lowerCorner, upperCorner,
+ updateGroupsCog, atomStart, atomEnd, atomDensity,
+ atinfo, x, numAtomsMoved, move, nbat);
+
+ cycleCounting_.stop(enbsCCgrid);
+ };
+
+ /* \brief Constructor
+ *
+ * \param[in] ePBC The periodic boundary conditions
+ * \param[in] numDDCells The number of domain decomposition cells per dimension, without DD nullptr should be passed
+ * \param[in] zones The domain decomposition zone setup, without DD nullptr should be passed
+ * \param[in] haveFep Tells whether non-bonded interactions are perturbed
+ * \param[in] maxNumThreads The maximum number of threads used in the search
+ */
+ PairSearch(int ePBC,
+ const ivec *numDDCells,
+ const gmx_domdec_zones_t *zones,
+ PairlistType pairlistType,
+ bool haveFep,
+ int maxNumthreads);
+
+ //! Sets the order of the local atoms to the order grid atom ordering
+ void setLocalAtomOrder()
+ {
+ gridSet_.setLocalAtomOrder();
+ }
+
+ const DomainSetup domainSetup() const
+ {
+ return domainSetup_;
+ }
+
+ //! Returns the set of search grids
+ const Nbnxm::GridSet &gridSet() const
+ {
+ return gridSet_;
+ }
+
+ //! Returns the list of thread-local work objects
+ gmx::ArrayRef<const PairsearchWork> work() const
+ {
+ return work_;
+ }
+
+ //! Returns the list of thread-local work objects
+ gmx::ArrayRef<PairsearchWork> work()
+ {
+ return work_;
+ }
+
+ private:
+ //! The domain setup
+ DomainSetup domainSetup_;
+ //! The set of search grids
+ Nbnxm::GridSet gridSet_;
+ //! Work objects, one entry for each thread
+ std::vector<PairsearchWork> work_;
+
+ public:
+ //! Cycle counting for measuring components of the search
+ SearchCycleCounting cycleCounting_;
};
-
-/*! \brief Start an nbnxn cycle counter */
-static inline void nbs_cycle_start(nbnxn_cycle_t *cc)
-{
- cc->start = gmx_cycles_read();
-}
-
-/*! \brief Stop an nbnxn cycle counter */
-static inline void nbs_cycle_stop(nbnxn_cycle_t *cc)
-{
- cc->c += gmx_cycles_read() - cc->start;
- cc->count++;
-}
-
#endif
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2019, by the GROMACS development team, led by
+ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+ * and including many others, as listed in the AUTHORS file in the
+ * top-level source directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/*! \internal \file
+ * \brief
+ * Implements the Nbnxm class
+ *
+ * \author Berk Hess <hess@kth.se>
+ * \ingroup module_nbnxm
+ */
+
+#include "gmxpre.h"
+
+#include "nbnxm.h"
+
+#include "gromacs/domdec/domdec_struct.h"
+#include "gromacs/nbnxm/atomdata.h"
+
+#include "internal.h"
+
+/*! \cond INTERNAL */
+
+void nbnxn_put_on_grid(nonbonded_verlet_t *nb_verlet,
+ const matrix box,
+ int ddZone,
+ const rvec lowerCorner,
+ const rvec upperCorner,
+ const gmx::UpdateGroupsCog *updateGroupsCog,
+ int atomStart,
+ int atomEnd,
+ real atomDensity,
+ const int *atinfo,
+ gmx::ArrayRef<const gmx::RVec> x,
+ int numAtomsMoved,
+ const int *move)
+{
+ nb_verlet->pairSearch_->putOnGrid(box, ddZone, lowerCorner, upperCorner,
+ updateGroupsCog, atomStart, atomEnd, atomDensity,
+ atinfo, x, numAtomsMoved, move,
+ nb_verlet->nbat.get());
+}
+
+/* Calls nbnxn_put_on_grid for all non-local domains */
+void nbnxn_put_on_grid_nonlocal(nonbonded_verlet_t *nbv,
+ const struct gmx_domdec_zones_t *zones,
+ const int *atinfo,
+ gmx::ArrayRef<const gmx::RVec> x)
+{
+ for (int zone = 1; zone < zones->n; zone++)
+ {
+ rvec c0, c1;
+ for (int d = 0; d < DIM; d++)
+ {
+ c0[d] = zones->size[zone].bb_x0[d];
+ c1[d] = zones->size[zone].bb_x1[d];
+ }
+
+ nbnxn_put_on_grid(nbv, nullptr,
+ zone, c0, c1,
+ nullptr,
+ zones->cg_range[zone],
+ zones->cg_range[zone+1],
+ -1,
+ atinfo,
+ x,
+ 0, nullptr);
+ }
+}
+
+gmx::ArrayRef<const int> nonbonded_verlet_t::getLocalAtomOrder() const
+{
+ /* Return the atom order for the home cell (index 0) */
+ const Nbnxm::Grid &grid = pairSearch_->gridSet().grids()[0];
+
+ const int numIndices = grid.atomIndexEnd() - grid.firstAtomInColumn(0);
+
+ return gmx::constArrayRefFromArray(pairSearch_->gridSet().atomIndices().data(), numIndices);
+}
+
+void nonbonded_verlet_t::setLocalAtomOrder()
+{
+ pairSearch_->setLocalAtomOrder();
+}
+
+void nonbonded_verlet_t::setAtomProperties(const t_mdatoms &mdatoms,
+ const int &atinfo)
+{
+ nbnxn_atomdata_set(nbat.get(), *pairSearch_, &mdatoms, &atinfo);
+}
+
+void nonbonded_verlet_t::setCoordinates(const Nbnxm::AtomLocality locality,
+ const bool fillLocal,
+ gmx::ArrayRef<const gmx::RVec> x,
+ gmx_wallcycle *wcycle)
+{
+ nbnxn_atomdata_copy_x_to_nbat_x(*pairSearch_, locality, fillLocal,
+ as_rvec_array(x.data()),
+ nbat.get(), wcycle);
+}
+
+void nonbonded_verlet_t::getLocalNumCells(int *numCellsX,
+ int *numCellsY) const
+{
+ pairSearch_->gridSet().getLocalNumCells(numCellsX, numCellsY);
+}
+
+gmx::ArrayRef<const int> nonbonded_verlet_t::getGridIndices() const
+{
+ return pairSearch_->gridSet().cells();
+}
+
+/*! \endcond */
struct gmx_wallcycle;
struct interaction_const_t;
struct nbnxn_pairlist_set_t;
-struct nbnxn_search;
struct nonbonded_verlet_t;
enum class PairlistType;
+class PairSearch;
struct t_blocka;
struct t_commrec;
struct t_lambda;
//! Construct the pairlist set for the given locality
void construct(Nbnxm::InteractionLocality iLocality,
- nbnxn_search *nbs,
+ PairSearch *pairSearch,
nbnxn_atomdata_t *nbat,
const t_blocka *excl,
Nbnxm::KernelType kernelbType,
};
//! Constructs an object from its components
- nonbonded_verlet_t(std::unique_ptr<PairlistSets> pairlistSets_,
- std::unique_ptr<nbnxn_search> nbs,
+ nonbonded_verlet_t(std::unique_ptr<PairlistSets> pairlistSets,
+ std::unique_ptr<PairSearch> pairSearch,
std::unique_ptr<nbnxn_atomdata_t> nbat,
const Nbnxm::KernelSetup &kernelSetup,
gmx_nbnxn_gpu_t *gpu_nbv);
//! Initialize the pair list sets, TODO this should be private
void initPairlistSets(bool haveMultipleDomains);
+ //! Returns the order of the local atoms on the grid
+ gmx::ArrayRef<const int> getLocalAtomOrder() const;
+
//! Sets the order of the local atoms to the order grid atom ordering
void setLocalAtomOrder();
+ //! Returns the index position of the atoms on the search grid
+ gmx::ArrayRef<const int> getGridIndices() const;
+
//! Constructs the pairlist for the given locality
void constructPairlist(Nbnxm::InteractionLocality iLocality,
const t_blocka *excl,
int64_t step,
t_nrnb *nrnb);
+ //! Updates all the atom properties in Nbnxm
+ void setAtomProperties(const t_mdatoms &mdatoms,
+ const int &atinfo);
+
+ //! Updates the coordinates in Nbnxm for the given locality
+ void setCoordinates(Nbnxm::AtomLocality locality,
+ bool fillLocal,
+ gmx::ArrayRef<const gmx::RVec> x,
+ gmx_wallcycle *wcycle);
+
//! Returns a reference to the pairlist sets
const PairlistSets &pairlistSets() const
{
//! All data related to the pair lists
std::unique_ptr<PairlistSets> pairlistSets_;
//! Working data for constructing the pairlists
- std::unique_ptr<nbnxn_search> nbs;
+ std::unique_ptr<PairSearch> pairSearch_;
//! Atom data
std::unique_ptr<nbnxn_atomdata_t> nbat;
private:
const int *atinfo,
gmx::ArrayRef<const gmx::RVec> x);
-/*! \brief Returns the order indices of the atoms on the pairlist search grid */
-gmx::ArrayRef<const int> nbnxn_get_atomorder(const nbnxn_search* nbs);
-
-/*! \brief Renumbers the atom indices on the grid to consecutive order */
-void nbnxn_set_atomorder(nbnxn_search *nbs);
-
-/*! \brief Returns the index position of the atoms on the pairlist search grid */
-gmx::ArrayRef<const int> nbnxn_get_gridindices(const nbnxn_search* nbs);
-
-#endif // GMX_NBNXN_NBNXN_H
+#endif // GMX_NBNXN_NBNXM_H
enbnxninitcombrule = enbnxninitcombruleNONE;
}
- std::unique_ptr<nbnxn_atomdata_t> nbat =
+ auto nbat =
std::make_unique<nbnxn_atomdata_t>(useGpu ? gmx::PinningPolicy::PinnedIfSupported : gmx::PinningPolicy::CannotBePinned);
int mimimumNumEnergyGroupNonbonded = ir->opts.ngener;
minimumIlistCountForGpuBalancing = getMinimumIlistCountForGpuBalancing(gpu_nbv);
}
- std::unique_ptr<nonbonded_verlet_t::PairlistSets> pairlistSets =
+ auto pairlistSets =
std::make_unique<nonbonded_verlet_t::PairlistSets>(listParams,
haveMultipleDomains,
minimumIlistCountForGpuBalancing);
- std::unique_ptr<nbnxn_search> nbs =
- std::make_unique<nbnxn_search>(ir->ePBC,
- DOMAINDECOMP(cr) ? &cr->dd->nc : nullptr,
- DOMAINDECOMP(cr) ? domdec_zones(cr->dd) : nullptr,
- listParams.pairlistType,
- bFEP_NonBonded,
- gmx_omp_nthreads_get(emntPairsearch));
+ auto pairSearch =
+ std::make_unique<PairSearch>(ir->ePBC,
+ DOMAINDECOMP(cr) ? &cr->dd->nc : nullptr,
+ DOMAINDECOMP(cr) ? domdec_zones(cr->dd) : nullptr,
+ listParams.pairlistType,
+ bFEP_NonBonded,
+ gmx_omp_nthreads_get(emntPairsearch));
return std::make_unique<nonbonded_verlet_t>(std::move(pairlistSets),
- std::move(nbs),
+ std::move(pairSearch),
std::move(nbat),
kernelSetup,
gpu_nbv);
} // namespace Nbnxm
nonbonded_verlet_t::nonbonded_verlet_t(std::unique_ptr<PairlistSets> pairlistSets,
- std::unique_ptr<nbnxn_search> nbs_in,
+ std::unique_ptr<PairSearch> pairSearch,
std::unique_ptr<nbnxn_atomdata_t> nbat_in,
const Nbnxm::KernelSetup &kernelSetup,
gmx_nbnxn_gpu_t *gpu_nbv_ptr) :
pairlistSets_(std::move(pairlistSets)),
- nbs(std::move(nbs_in)),
+ pairSearch_(std::move(pairSearch)),
nbat(std::move(nbat_in)),
kernelSetup_(kernelSetup),
gpu_nbv(gpu_nbv_ptr)
{
GMX_RELEASE_ASSERT(pairlistSets_, "Need valid pairlistSets");
- GMX_RELEASE_ASSERT(nbs, "Need valid search object");
+ GMX_RELEASE_ASSERT(pairSearch_, "Need valid search object");
GMX_RELEASE_ASSERT(nbat, "Need valid atomdata object");
}
constexpr bool c_pbcShiftBackward = true;
-static void nbs_cycle_clear(nbnxn_cycle_t *cc)
-{
- for (int i = 0; i < enbsCCnr; i++)
- {
- cc[i].count = 0;
- cc[i].c = 0;
- }
-}
-
-static double Mcyc_av(const nbnxn_cycle_t *cc)
-{
- return static_cast<double>(cc->c)*1e-6/cc->count;
-}
-
-static void nbs_cycle_print(FILE *fp, const nbnxn_search *nbs)
+void PairSearch::SearchCycleCounting::printCycles(FILE *fp,
+ gmx::ArrayRef<const PairsearchWork> work) const
{
fprintf(fp, "\n");
- fprintf(fp, "ns %4d grid %4.1f search %4.1f red.f %5.3f",
- nbs->cc[enbsCCgrid].count,
- Mcyc_av(&nbs->cc[enbsCCgrid]),
- Mcyc_av(&nbs->cc[enbsCCsearch]),
- Mcyc_av(&nbs->cc[enbsCCreducef]));
+ fprintf(fp, "ns %4d grid %4.1f search %4.1f",
+ cc_[enbsCCgrid].count(),
+ cc_[enbsCCgrid].averageMCycles(),
+ cc_[enbsCCsearch].averageMCycles());
- if (nbs->work.size() > 1)
+ if (work.size() > 1)
{
- if (nbs->cc[enbsCCcombine].count > 0)
+ if (cc_[enbsCCcombine].count() > 0)
{
fprintf(fp, " comb %5.2f",
- Mcyc_av(&nbs->cc[enbsCCcombine]));
+ cc_[enbsCCcombine].averageMCycles());
}
fprintf(fp, " s. th");
- for (const nbnxn_search_work_t &work : nbs->work)
+ for (const PairsearchWork &workEntry : work)
{
fprintf(fp, " %4.1f",
- Mcyc_av(&work.cc[enbsCCsearch]));
+ workEntry.cycleCounter.averageMCycles());
}
}
fprintf(fp, "\n");
sfree(nl->excl_fep);
}
-nbnxn_search_work_t::nbnxn_search_work_t() :
+PairsearchWork::PairsearchWork() :
cp0({{0}}
),
buffer_flags({0, nullptr, 0}),
cp1({{0}})
{
nbnxn_init_pairlist_fep(nbl_fep.get());
-
- nbs_cycle_clear(cc);
}
-nbnxn_search_work_t::~nbnxn_search_work_t()
+PairsearchWork::~PairsearchWork()
{
sfree(buffer_flags.flag);
free_nblist(nbl_fep.get());
}
-nbnxn_search::DomainSetup::DomainSetup(const int ePBC,
- const ivec *numDDCells,
- const gmx_domdec_zones_t *ddZones) :
+// TODO: Move to pairsearch.cpp
+PairSearch::DomainSetup::DomainSetup(const int ePBC,
+ const ivec *numDDCells,
+ const gmx_domdec_zones_t *ddZones) :
ePBC(ePBC),
haveDomDec(numDDCells != nullptr),
zones(ddZones)
}
}
-nbnxn_search::nbnxn_search(const int ePBC,
- const ivec *numDDCells,
- const gmx_domdec_zones_t *ddZones,
- const PairlistType pairlistType,
- const bool haveFep,
- const int maxNumThreads) :
+// TODO: Move to pairsearch.cpp
+PairSearch::PairSearch(const int ePBC,
+ const ivec *numDDCells,
+ const gmx_domdec_zones_t *ddZones,
+ const PairlistType pairlistType,
+ const bool haveFep,
+ const int maxNumThreads) :
domainSetup_(ePBC, numDDCells, ddZones),
gridSet_(domainSetup_.haveDomDecPerDim, pairlistType, haveFep, maxNumThreads),
- search_count(0),
- work(maxNumThreads)
+ work_(maxNumThreads)
{
- print_cycles = (getenv("GMX_NBNXN_CYCLE") != nullptr);
- nbs_cycle_clear(cc);
+ cycleCounting_.recordCycles_ = (getenv("GMX_NBNXN_CYCLE") != nullptr);
}
static void init_buffer_flags(nbnxn_buffer_flags_t *flags,
}
/* Print statistics of a pair list, used for debug output */
-static void print_nblist_statistics(FILE *fp, const NbnxnPairlistCpu *nbl,
- const nbnxn_search *nbs, real rl)
+static void print_nblist_statistics(FILE *fp,
+ const NbnxnPairlistCpu *nbl,
+ const PairSearch &pairSearch,
+ const real rl)
{
- const Grid &grid = nbs->gridSet().grids()[0];
+ const Grid &grid = pairSearch.gridSet().grids()[0];
const Grid::Dimensions &dims = grid.dimensions();
fprintf(fp, "nbl nci %zu ncj %d\n",
}
/* Print statistics of a pair lists, used for debug output */
-static void print_nblist_statistics(FILE *fp, const NbnxnPairlistGpu *nbl,
- const nbnxn_search *nbs, real rl)
+static void print_nblist_statistics(FILE *fp,
+ const NbnxnPairlistGpu *nbl,
+ const PairSearch &pairSearch,
+ const real rl)
{
- const Grid &grid = nbs->gridSet().grids()[0];
+ const Grid &grid = pairSearch.gridSet().grids()[0];
const Grid::Dimensions &dims = grid.dimensions();
fprintf(fp, "nbl nsci %zu ncj4 %zu nsi %d excl4 %zu\n",
}
/* Estimates the average size of a full j-list for super/sub setup */
-static void get_nsubpair_target(const nbnxn_search *nbs,
+static void get_nsubpair_target(const PairSearch &pairSearch,
const InteractionLocality iloc,
const real rlist,
const int min_ci_balanced,
const int nsubpair_target_min = 36;
real r_eff_sup, vol_est, nsp_est, nsp_est_nl;
- const Grid &grid = nbs->gridSet().grids()[0];
+ const Grid &grid = pairSearch.gridSet().grids()[0];
/* We don't need to balance list sizes if:
* - We didn't request balancing.
/* The formulas below are a heuristic estimate of the average nsj per si*/
r_eff_sup = rlist + nbnxn_get_rlist_effective_inc(numAtomsCluster, ls);
- if (!nbs->domainSetup().haveDomDec || nbs->domainSetup().zones->n == 1)
+ if (!pairSearch.domainSetup().haveDomDec ||
+ pairSearch.domainSetup().zones->n == 1)
{
nsp_est_nl = 0;
}
{
nsp_est_nl =
gmx::square(dims.atomDensity/numAtomsCluster)*
- nonlocal_vol2(nbs->domainSetup().zones, ls, r_eff_sup);
+ nonlocal_vol2(pairSearch.domainSetup().zones, ls, r_eff_sup);
}
if (iloc == InteractionLocality::Local)
}
}
-static void balance_fep_lists(const nbnxn_search *nbs,
- nbnxn_pairlist_set_t *nbl_lists)
+static void balance_fep_lists(gmx::ArrayRef<PairsearchWork> work,
+ nbnxn_pairlist_set_t *nbl_lists)
{
int nnbl;
int nri_tot, nrj_tot, nrj_target;
{
try
{
- t_nblist *nbl = nbs->work[th].nbl_fep.get();
+ t_nblist *nbl = work[th].nbl_fep.get();
/* Note that here we allocate for the total size, instead of
* a per-thread esimate (which is hard to obtain).
/* Loop over the source lists and assign and copy i-entries */
th_dest = 0;
- nbld = nbs->work[th_dest].nbl_fep.get();
+ nbld = work[th_dest].nbl_fep.get();
for (int th = 0; th < nnbl; th++)
{
t_nblist *nbls;
nbld->nrj + nrj - nrj_target > nrj_target - nbld->nrj)
{
th_dest++;
- nbld = nbs->work[th_dest].nbl_fep.get();
+ nbld = work[th_dest].nbl_fep.get();
}
nbld->iinr[nbld->nri] = nbls->iinr[i];
/* Swap the list pointers */
for (int th = 0; th < nnbl; th++)
{
- t_nblist *nbl_tmp = nbs->work[th].nbl_fep.release();
- nbs->work[th].nbl_fep.reset(nbl_lists->nbl_fep[th]);
+ t_nblist *nbl_tmp = work[th].nbl_fep.release();
+ work[th].nbl_fep.reset(nbl_lists->nbl_fep[th]);
nbl_lists->nbl_fep[th] = nbl_tmp;
if (debug)
/* Generates the part of pair-list nbl assigned to our thread */
template <typename T>
-static void nbnxn_make_pairlist_part(const nbnxn_search *nbs,
+static void nbnxn_make_pairlist_part(const PairSearch &pairSearch,
const Grid &iGrid,
const Grid &jGrid,
- nbnxn_search_work_t *work,
+ PairsearchWork *work,
const nbnxn_atomdata_t *nbat,
const t_blocka &exclusions,
real rlist,
gmx_bitmask_t *gridj_flag = nullptr;
int ncj_old_i, ncj_old_j;
- nbs_cycle_start(&work->cc[enbsCCsearch]);
-
if (jGrid.geometry().isSimple != pairlistIsSimple(*nbl) ||
iGrid.geometry().isSimple != pairlistIsSimple(*nbl))
{
gridj_flag = work->buffer_flags.flag;
}
- const Nbnxm::GridSet &gridSet = nbs->gridSet();
+ const Nbnxm::GridSet &gridSet = pairSearch.gridSet();
gridSet.getBox(box);
/* Check if we need periodicity shifts.
* Without PBC or with domain decomposition we don't need them.
*/
- if (d >= ePBC2npbcdim(nbs->domainSetup().ePBC) ||
- nbs->domainSetup().haveDomDecPerDim[d])
+ if (d >= ePBC2npbcdim(pairSearch.domainSetup().ePBC) ||
+ pairSearch.domainSetup().haveDomDecPerDim[d])
{
shp[d] = 0;
}
work->ndistc = numDistanceChecks;
- nbs_cycle_stop(&work->cc[enbsCCsearch]);
-
checkListSizeConsistency(*nbl, haveFep);
if (debug)
{
fprintf(debug, "number of distance checks %d\n", numDistanceChecks);
- print_nblist_statistics(debug, nbl, nbs, rlist);
+ print_nblist_statistics(debug, nbl, pairSearch, rlist);
if (haveFep)
{
}
}
-static void reduce_buffer_flags(const nbnxn_search *nbs,
+static void reduce_buffer_flags(const PairSearch &pairSearch,
int nsrc,
const nbnxn_buffer_flags_t *dest)
{
for (int s = 0; s < nsrc; s++)
{
- gmx_bitmask_t * flag = nbs->work[s].buffer_flags.flag;
+ gmx_bitmask_t * flag = pairSearch.work()[s].buffer_flags.flag;
for (int b = 0; b < dest->nflag; b++)
{
static void rebalanceSimpleLists(int numLists,
NbnxnPairlistCpu * const * const srcSet,
NbnxnPairlistCpu **destSet,
- gmx::ArrayRef<nbnxn_search_work_t> searchWork)
+ gmx::ArrayRef<PairsearchWork> searchWork)
{
int ncjTotal = 0;
for (int s = 0; s < numLists; s++)
void
nonbonded_verlet_t::PairlistSets::construct(const InteractionLocality iLocality,
- nbnxn_search *nbs,
+ PairSearch *pairSearch,
nbnxn_atomdata_t *nbat,
const t_blocka *excl,
const Nbnxm::KernelType kernelType,
}
else
{
- nzi = nbs->domainSetup().zones->nizone;
+ nzi = pairSearch->domainSetup().zones->nizone;
}
if (!nbl_list->bSimple && minimumIlistCountForGpuBalancing_ > 0)
{
- get_nsubpair_target(nbs, iLocality, rlist, minimumIlistCountForGpuBalancing_,
+ get_nsubpair_target(*pairSearch, iLocality, rlist, minimumIlistCountForGpuBalancing_,
&nsubpair_target, &nsubpair_tot_est);
}
else
clear_pairlist(nbl_list->nblGpu[th]);
}
- if (nbs->gridSet().haveFep())
+ if (pairSearch->gridSet().haveFep())
{
clear_pairlist_fep(nbl_list->nbl_fep[th]);
}
}
- const gmx_domdec_zones_t *ddZones = nbs->domainSetup().zones;
+ const gmx_domdec_zones_t *ddZones = pairSearch->domainSetup().zones;
for (int zi = 0; zi < nzi; zi++)
{
- const Grid &iGrid = nbs->gridSet().grids()[zi];
+ const Grid &iGrid = pairSearch->gridSet().grids()[zi];
int zj0;
int zj1;
}
for (int zj = zj0; zj < zj1; zj++)
{
- const Grid &jGrid = nbs->gridSet().grids()[zj];
+ const Grid &jGrid = pairSearch->gridSet().grids()[zj];
if (debug)
{
fprintf(debug, "ns search grid %d vs %d\n", zi, zj);
}
- nbs_cycle_start(&nbs->cc[enbsCCsearch]);
+ pairSearch->cycleCounting_.start(PairSearch::enbsCCsearch);
- ci_block = get_ci_block_size(iGrid, nbs->domainSetup().haveDomDec, nnbl);
+ ci_block = get_ci_block_size(iGrid, pairSearch->domainSetup().haveDomDec, nnbl);
/* With GPU: generate progressively smaller lists for
* load balancing for local only or non-local with 2 zones.
*/
if (nbat->bUseBufferFlags && ((zi == 0 && zj == 0)))
{
- init_buffer_flags(&nbs->work[th].buffer_flags, nbat->numAtoms());
+ init_buffer_flags(&pairSearch->work()[th].buffer_flags, nbat->numAtoms());
}
if (CombineNBLists && th > 0)
clear_pairlist(nbl_list->nblGpu[th]);
}
+ auto &searchWork = pairSearch->work()[th];
+
+ searchWork.cycleCounter.start();
+
/* Divide the i super cell equally over the nblists */
if (nbl_list->bSimple)
{
- nbnxn_make_pairlist_part(nbs, iGrid, jGrid,
- &nbs->work[th], nbat, *excl,
+ nbnxn_make_pairlist_part(*pairSearch, iGrid, jGrid,
+ &searchWork, nbat, *excl,
rlist,
kernelType,
ci_block,
}
else
{
- nbnxn_make_pairlist_part(nbs, iGrid, jGrid,
- &nbs->work[th], nbat, *excl,
+ nbnxn_make_pairlist_part(*pairSearch, iGrid, jGrid,
+ &searchWork, nbat, *excl,
rlist,
kernelType,
ci_block,
nbl_list->nblGpu[th],
nbl_list->nbl_fep[th]);
}
+
+ searchWork.cycleCounter.stop();
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
}
- nbs_cycle_stop(&nbs->cc[enbsCCsearch]);
+ pairSearch->cycleCounting_.stop(PairSearch::enbsCCsearch);
np_tot = 0;
np_noq = 0;
np_hlj = 0;
for (int th = 0; th < nnbl; th++)
{
- inc_nrnb(nrnb, eNR_NBNXN_DIST2, nbs->work[th].ndistc);
+ inc_nrnb(nrnb, eNR_NBNXN_DIST2, pairSearch->work()[th].ndistc);
if (nbl_list->bSimple)
{
GMX_ASSERT(!nbl_list->bSimple, "Can only combine GPU lists");
NbnxnPairlistGpu **nbl = nbl_list->nblGpu;
- nbs_cycle_start(&nbs->cc[enbsCCcombine]);
+ pairSearch->cycleCounting_.start(PairSearch::enbsCCcombine);
combine_nblists(nnbl-1, nbl+1, nbl[0]);
- nbs_cycle_stop(&nbs->cc[enbsCCcombine]);
+ pairSearch->cycleCounting_.stop(PairSearch::enbsCCcombine);
}
}
}
{
if (nnbl > 1 && checkRebalanceSimpleLists(nbl_list))
{
- rebalanceSimpleLists(nbl_list->nnbl, nbl_list->nbl, nbl_list->nbl_work, nbs->work);
+ rebalanceSimpleLists(nbl_list->nnbl, nbl_list->nbl, nbl_list->nbl_work, pairSearch->work());
/* Swap the pointer of the sets of pair lists */
NbnxnPairlistCpu **tmp = nbl_list->nbl;
if (nbat->bUseBufferFlags)
{
- reduce_buffer_flags(nbs, nbl_list->nnbl, &nbat->buffer_flags);
+ reduce_buffer_flags(*pairSearch, nbl_list->nnbl, &nbat->buffer_flags);
}
- if (nbs->gridSet().haveFep())
+ if (pairSearch->gridSet().haveFep())
{
/* Balance the free-energy lists over all the threads */
- balance_fep_lists(nbs, nbl_list);
+ balance_fep_lists(pairSearch->work(), nbl_list);
}
if (nbl_list->bSimple)
/* Special performance logging stuff (env.var. GMX_NBNXN_CYCLE) */
if (iLocality == InteractionLocality::Local)
{
- nbs->search_count++;
+ pairSearch->cycleCounting_.searchCount_++;
}
- if (nbs->print_cycles &&
- (!nbs->domainSetup().haveDomDec || iLocality == InteractionLocality::NonLocal) &&
- nbs->search_count % 100 == 0)
+ if (pairSearch->cycleCounting_.recordCycles_ &&
+ (!pairSearch->domainSetup().haveDomDec || iLocality == InteractionLocality::NonLocal) &&
+ pairSearch->cycleCounting_.searchCount_ % 100 == 0)
{
- nbs_cycle_print(stderr, nbs);
+ pairSearch->cycleCounting_.printCycles(stderr, pairSearch->work());
}
/* If we have more than one list, they either got rebalancing (CPU)
{
for (int t = 0; t < nbl_list->nnbl; t++)
{
- print_nblist_statistics(debug, nbl_list->nbl[t], nbs, rlist);
+ print_nblist_statistics(debug, nbl_list->nbl[t], *pairSearch, rlist);
}
}
else
{
- print_nblist_statistics(debug, nbl_list->nblGpu[0], nbs, rlist);
+ print_nblist_statistics(debug, nbl_list->nblGpu[0], *pairSearch, rlist);
}
}
int64_t step,
t_nrnb *nrnb)
{
- pairlistSets_->construct(iLocality, nbs.get(), nbat.get(), excl,
+ pairlistSets_->construct(iLocality, pairSearch_.get(), nbat.get(), excl,
kernelSetup_.kernelType,
step, nrnb);
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff