From d380efb0df97143449c75d18dd73704cdb3fa719 Mon Sep 17 00:00:00 2001 From: Berk Hess Date: Wed, 19 Sep 2018 12:51:38 +0200 Subject: [PATCH] Enable update groups Also changed the naming of the decomposition unit. Everything was named charge group, now it can also be atom or atom group. Update groups are now mentioned in the manual and release notes Change-Id: Ide81a759aa623be22672583dd5a8bac332c4c904 --- .../parallelization-domain-decomp.rst | 17 ++++++--- docs/release-notes/performance.rst | 9 +++++ src/gromacs/domdec/domdec.cpp | 24 +++++++++---- src/gromacs/domdec/domdec_topology.cpp | 2 +- src/gromacs/domdec/partition.cpp | 5 +-- src/gromacs/domdec/redistribute.cpp | 36 +++++++++++-------- src/gromacs/mdlib/lincs.cpp | 2 +- 7 files changed, 64 insertions(+), 31 deletions(-) diff --git a/docs/reference-manual/algorithms/parallelization-domain-decomp.rst b/docs/reference-manual/algorithms/parallelization-domain-decomp.rst index 02d89c770b..76b862a545 100644 --- a/docs/reference-manual/algorithms/parallelization-domain-decomp.rst +++ b/docs/reference-manual/algorithms/parallelization-domain-decomp.rst @@ -35,11 +35,18 @@ In the most general case of a triclinic unit cell, the space in divided with a 1-, 2-, or 3-D grid in parallelepipeds that we call domain decomposition cells. Each cell is assigned to a particle-particle rank. The system is partitioned over the ranks at the beginning of each MD -step in which neighbor searching is performed. Since the neighbor -searching is based on charge groups, charge groups are also the units -for the domain decomposition. Charge groups are assigned to the cell -where their center of geometry resides. Before the forces can be -calculated, the coordinates from some neighboring cells need to be +step in which neighbor searching is performed. The minimum unit of +partitioning can be an atom, or a charge group with the (deprecated) +group cut-off scheme or an update group. An update group is a group +of atoms that has dependencies during update, which occurs when using +constraints and/or virtual sites. Thus different update groups can be +updated independenly. Currently update groups can only be used with at most +two sequential constraints, which is the case when only constraining +bonds involving hydrogen atoms. The advantages of update groups are that +no communication is required in the update and that this allows updating part +of the system while computing forces for other parts. Atom groups are assigned +to the cell where their center of geometry resides. Before the forces can +be calculated, the coordinates from some neighboring cells need to be communicated, and after the forces are calculated, the forces need to be communicated in the other direction. The communication and force assignment is based on zones that can cover one or multiple cells. An diff --git a/docs/release-notes/performance.rst b/docs/release-notes/performance.rst index 3bbff7ba1a..fb5e87151d 100644 --- a/docs/release-notes/performance.rst +++ b/docs/release-notes/performance.rst @@ -1,6 +1,15 @@ Performance improvements ^^^^^^^^^^^^^^^^^^^^^^^^ +Implemented update groups +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Domain decomposition can now be based on so-called update groups. These +are groups of atoms with dependencies during the update, which can be +constraints and virtual sites. Update groups can typically be used when +only bonds involving hydrogens are constrained and are enabled +automatically when possible. This improves performance by eliminating +MPI and OpenMP communication for constraints and virtual sites. + PME on GPU when running free energy perturbations not involving charges """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" PME can now be run on a GPU when doing free energy perturbations diff --git a/src/gromacs/domdec/domdec.cpp b/src/gromacs/domdec/domdec.cpp index 5f556966e4..ec57e3ff01 100644 --- a/src/gromacs/domdec/domdec.cpp +++ b/src/gromacs/domdec/domdec.cpp @@ -108,7 +108,7 @@ static const char *edlbs_names[int(DlbState::nr)] = { "off", "auto", "locked", "on", "on" }; -/* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */ +/* The size per atom group of the cggl_flag buffer in gmx_domdec_comm_t */ #define DD_CGIBS 2 /* The flags for the cggl_flag buffer in gmx_domdec_comm_t */ @@ -2112,8 +2112,6 @@ static void setupUpdateGroups(const gmx::MDLogger &mdlog, * should be compatible with the box size. */ comm->useUpdateGroups = (atomToAtomIntoDomainToDomainCutoff(*comm, 0) < cutoffMargin); - /* TODO: Enable update groups when all infrastructure is present */ - comm->useUpdateGroups = false; if (comm->useUpdateGroups) { @@ -2125,9 +2123,7 @@ static void setupUpdateGroups(const gmx::MDLogger &mdlog, } else { - /* TODO: Remove this comment when enabling update groups - * GMX_LOG(mdlog.info).appendTextFormatted("The combination of rlist and box size prohibits the use of update groups\n"); - */ + GMX_LOG(mdlog.info).appendTextFormatted("The combination of rlist and box size prohibits the use of update groups\n"); comm->updateGroupingPerMoleculetype.clear(); comm->updateGroupsCog.reset(nullptr); } @@ -2661,7 +2657,21 @@ static void writeSettings(gmx::TextWriter *log, bInterCGVsites || dd->splitConstraints || dd->splitSettles) { - log->writeLine("The maximum allowed distance for charge groups involved in interactions is:"); + std::string decompUnits; + if (comm->bCGs) + { + decompUnits = "charge groups"; + } + else if (comm->useUpdateGroups) + { + decompUnits = "atom groups"; + } + else + { + decompUnits = "atoms"; + } + + log->writeLineFormatted("The maximum allowed distance for %s involved in interactions is:", decompUnits.c_str()); log->writeLineFormatted("%40s %-7s %6.3f nm", "non-bonded interactions", "", comm->cutoff); if (bDynLoadBal) diff --git a/src/gromacs/domdec/domdec_topology.cpp b/src/gromacs/domdec/domdec_topology.cpp index e553e87f3f..512860bc54 100644 --- a/src/gromacs/domdec/domdec_topology.cpp +++ b/src/gromacs/domdec/domdec_topology.cpp @@ -2694,7 +2694,7 @@ void dd_bonded_cg_distance(const gmx::MDLogger &mdlog, if (*r_2b > 0 || *r_mb > 0) { - GMX_LOG(mdlog.info).appendText("Initial maximum inter charge-group distances:"); + GMX_LOG(mdlog.info).appendText("Initial maximum distances in bonded interactions:"); if (*r_2b > 0) { GMX_LOG(mdlog.info).appendTextFormatted( diff --git a/src/gromacs/domdec/partition.cpp b/src/gromacs/domdec/partition.cpp index 0fe65fcfb8..7d9e8b6cd9 100644 --- a/src/gromacs/domdec/partition.cpp +++ b/src/gromacs/domdec/partition.cpp @@ -2802,7 +2802,8 @@ static void dd_sort_state(gmx_domdec_t *dd, rvec *cgcm, t_forcerec *fr, t_state dd->ncg_home = sort->sorted.size(); if (debug) { - fprintf(debug, "Set the new home charge group count to %d\n", + fprintf(debug, "Set the new home %s count to %d\n", + dd->comm->bCGs ? "charge group" : "atom", dd->ncg_home); } @@ -3303,7 +3304,7 @@ void dd_partition_system(FILE *fplog, ncg_home_old = dd->ncg_home; - /* When repartitioning we mark charge groups that will move to neighboring + /* When repartitioning we mark atom groups that will move to neighboring * DD cells, but we do not move them right away for performance reasons. * Thus we need to keep track of how many charge groups will move for * obtaining correct local charge group / atom counts. diff --git a/src/gromacs/domdec/redistribute.cpp b/src/gromacs/domdec/redistribute.cpp index 3dcba3b73e..2e7920b9cd 100644 --- a/src/gromacs/domdec/redistribute.cpp +++ b/src/gromacs/domdec/redistribute.cpp @@ -166,25 +166,32 @@ static void print_cg_move(FILE *fplog, gmx_bool bHaveCgcmOld, real limitd, rvec cm_old, rvec cm_new, real pos_d) { - gmx_domdec_comm_t *comm; - char buf[22]; + const gmx_domdec_comm_t *comm = dd->comm; + std::string mesg; - comm = dd->comm; + fprintf(fplog, "\nStep %" PRId64 ":\n", step); - fprintf(fplog, "\nStep %s:\n", gmx_step_str(step, buf)); - if (limitd > 0) + if (comm->bCGs) + { + mesg += "The charge group starting at atom"; + } + else if (comm->useUpdateGroups) { - fprintf(fplog, "%s %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n", - dd->comm->bCGs ? "The charge group starting at atom" : "Atom", - ddglatnr(dd, dd->atomGrouping().block(cg).begin()), limitd, dim2char(dim)); + mesg += "The update group starting at atom"; } else { - /* We don't have a limiting distance available: don't print it */ - fprintf(fplog, "%s %d moved more than the distance allowed by the domain decomposition in direction %c\n", - dd->comm->bCGs ? "The charge group starting at atom" : "Atom", - ddglatnr(dd, dd->atomGrouping().block(cg).begin()), dim2char(dim)); + mesg += "Atom"; } + mesg += gmx::formatString(" %d moved more than the distance allowed by the domain decomposition", + ddglatnr(dd, dd->atomGrouping().block(cg).begin())); + if (limitd > 0) + { + mesg += gmx::formatString(" (%f)", limitd); + } + mesg += gmx::formatString(" in direction %c\n", dim2char(dim)); + fprintf(fplog, "%s", mesg.c_str()); + fprintf(fplog, "distance out of cell %f\n", dir == 1 ? pos_d - comm->cell_x1[dim] : pos_d - comm->cell_x0[dim]); if (bHaveCgcmOld) @@ -217,9 +224,8 @@ static void cg_move_error(FILE *fplog, print_cg_move(stderr, dd, step, cg, dim, dir, bHaveCgcmOld, limitd, cm_old, cm_new, pos_d); gmx_fatal(FARGS, - "%s moved too far between two domain decomposition steps\n" - "This usually means that your system is not well equilibrated", - dd->comm->bCGs ? "An atom group" : "An atom"); + "One or more atoms moved too far between two domain decomposition steps.\n" + "This usually means that your system is not well equilibrated"); } static void rotate_state_atom(t_state *state, int a) diff --git a/src/gromacs/mdlib/lincs.cpp b/src/gromacs/mdlib/lincs.cpp index f57d901a03..cde84e0002 100644 --- a/src/gromacs/mdlib/lincs.cpp +++ b/src/gromacs/mdlib/lincs.cpp @@ -1599,7 +1599,7 @@ Lincs *init_lincs(FILE *fplog, const gmx_mtop_t &mtop, fprintf(fplog, "The number of constraints is %d\n", li->ncg); if (bPLINCS) { - fprintf(fplog, "There are inter charge-group constraints,\n" + fprintf(fplog, "There are constraints between atoms in different decomposition domains,\n" "will communicate selected coordinates each lincs iteration\n"); } if (li->ncg_triangle > 0) -- 2.22.0