From: Mark Abraham Date: Sat, 6 Feb 2016 01:54:15 +0000 (+0000) Subject: Merge branch release-5-1 X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=2d4cfab660160f3de2c3c32bb51896883d5d7e8c;p=alexxy%2Fgromacs.git Merge branch release-5-1 Moved new test/readinp.cpp file to fileio where readinp.cpp is now located. Left skeleton test infrastructure in place but deactivated until needed again. Change-Id: I9261aba261314170634322018b4315d2ebe35601 --- 2d4cfab660160f3de2c3c32bb51896883d5d7e8c diff --cc cmake/gmxVersionInfo.cmake index 56ca2feeb9,cf59cbd827..87b96f0d36 --- a/cmake/gmxVersionInfo.cmake +++ b/cmake/gmxVersionInfo.cmake @@@ -254,10 -256,10 +254,10 @@@ set(REGRESSIONTEST_BRANCH "refs/heads/m # each release. It's hard to test because it is only used for # REGRESSIONTEST_DOWNLOAD, which doesn't work until that tarball has # been placed on the server. - set(REGRESSIONTEST_MD5SUM "614a74e9b143bda5476f87f4ce08eec0" CACHE INTERNAL "MD5 sum of the regressiontests tarball") + set(REGRESSIONTEST_MD5SUM "3f663536649db883a5616f25f95ac927" CACHE INTERNAL "MD5 sum of the regressiontests tarball") math(EXPR GMX_VERSION_NUMERIC - "${GMX_VERSION_MAJOR}*10000 + ${GMX_VERSION_MINOR}*100 + ${GMX_VERSION_PATCH}") + "${GMX_VERSION_MAJOR}*10000 + ${GMX_VERSION_PATCH}") set(GMX_API_VERSION ${GMX_VERSION_NUMERIC}) ##################################################################### diff --cc src/gromacs/ewald/pme.cpp index 353cfa0221,649e3a2b6f..81db679a23 --- a/src/gromacs/ewald/pme.cpp +++ b/src/gromacs/ewald/pme.cpp @@@ -1090,97 -1089,94 +1091,97 @@@ int gmx_pme_do(struct gmx_pme_t *pme /* Here we start a large thread parallel region */ #pragma omp parallel num_threads(pme->nthread) private(thread) { - thread = gmx_omp_get_thread_num(); - if (flags & GMX_PME_SOLVE) + try { - int loop_count; - - /* do 3d-fft */ - if (thread == 0) - { - wallcycle_start(wcycle, ewcPME_FFT); - } - gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_REAL_TO_COMPLEX, - thread, wcycle); - if (thread == 0) - { - wallcycle_stop(wcycle, ewcPME_FFT); - } - where(); - - /* solve in k-space for our local cells */ - if (thread == 0) - { - wallcycle_start(wcycle, (grid_index < DO_Q ? ewcPME_SOLVE : ewcLJPME)); - } - if (grid_index < DO_Q) - { - loop_count = - solve_pme_yzx(pme, cfftgrid, ewaldcoeff_q, - box[XX][XX]*box[YY][YY]*box[ZZ][ZZ], - bCalcEnerVir, - pme->nthread, thread); - } - else + thread = gmx_omp_get_thread_num(); + if (flags & GMX_PME_SOLVE) { - loop_count = - solve_pme_lj_yzx(pme, &cfftgrid, FALSE, ewaldcoeff_lj, - box[XX][XX]*box[YY][YY]*box[ZZ][ZZ], - bCalcEnerVir, - pme->nthread, thread); - } + int loop_count; - if (thread == 0) - { - wallcycle_stop(wcycle, (grid_index < DO_Q ? ewcPME_SOLVE : ewcLJPME)); + /* do 3d-fft */ + if (thread == 0) + { + wallcycle_start(wcycle, ewcPME_FFT); + } + gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_REAL_TO_COMPLEX, + thread, wcycle); + if (thread == 0) + { + wallcycle_stop(wcycle, ewcPME_FFT); + } where(); - inc_nrnb(nrnb, eNR_SOLVEPME, loop_count); - } - } - if (bBackFFT) - { - /* do 3d-invfft */ - if (thread == 0) - { - where(); - wallcycle_start(wcycle, ewcPME_FFT); + /* solve in k-space for our local cells */ + if (thread == 0) + { + wallcycle_start(wcycle, (grid_index < DO_Q ? ewcPME_SOLVE : ewcLJPME)); + } + if (grid_index < DO_Q) + { + loop_count = + solve_pme_yzx(pme, cfftgrid, ewaldcoeff_q, + box[XX][XX]*box[YY][YY]*box[ZZ][ZZ], + bCalcEnerVir, + pme->nthread, thread); + } + else + { + loop_count = + solve_pme_lj_yzx(pme, &cfftgrid, FALSE, ewaldcoeff_lj, + box[XX][XX]*box[YY][YY]*box[ZZ][ZZ], + bCalcEnerVir, + pme->nthread, thread); + } + + if (thread == 0) + { + wallcycle_stop(wcycle, (grid_index < DO_Q ? ewcPME_SOLVE : ewcLJPME)); + where(); + inc_nrnb(nrnb, eNR_SOLVEPME, loop_count); + } } - gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_COMPLEX_TO_REAL, - thread, wcycle); - if (thread == 0) + - if (bCalcF) ++ if (bBackFFT) { - wallcycle_stop(wcycle, ewcPME_FFT); + /* do 3d-invfft */ + if (thread == 0) + { + where(); + wallcycle_start(wcycle, ewcPME_FFT); + } + gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_COMPLEX_TO_REAL, + thread, wcycle); + if (thread == 0) + { + wallcycle_stop(wcycle, ewcPME_FFT); - where(); + where(); - if (pme->nodeid == 0) - { - real ntot = pme->nkx*pme->nky*pme->nkz; - npme = static_cast(ntot*log(ntot)/log(2.0)); - inc_nrnb(nrnb, eNR_FFT, 2*npme); + if (pme->nodeid == 0) + { + real ntot = pme->nkx*pme->nky*pme->nkz; + npme = static_cast(ntot*log(ntot)/log(2.0)); + inc_nrnb(nrnb, eNR_FFT, 2*npme); + } + + /* Note: this wallcycle region is closed below + outside an OpenMP region, so take care if + refactoring code here. */ + wallcycle_start(wcycle, ewcPME_SPREADGATHER); } - /* Note: this wallcycle region is closed below - outside an OpenMP region, so take care if - refactoring code here. */ - wallcycle_start(wcycle, ewcPME_SPREADGATHER); + copy_fftgrid_to_pmegrid(pme, fftgrid, grid, grid_index, pme->nthread, thread); } - - copy_fftgrid_to_pmegrid(pme, fftgrid, grid, grid_index, pme->nthread, thread); - } + } GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; } /* End of thread parallel section. * With MPI we have to synchronize here before gmx_sum_qgrid_dd. */ - if (bCalcF) + if (bBackFFT) { /* distribute local grid to all nodes */ -#ifdef GMX_MPI +#if GMX_MPI if (pme->nnodes > 1) { gmx_sum_qgrid_dd(pme, grid, GMX_SUM_GRID_BACKWARD); @@@ -1475,26 -1459,25 +1479,31 @@@ unwrap_periodic_pmegrid(pme, grid); - /* interpolate forces for our local atoms */ - where(); - bClearF = (bFirst && PAR(cr)); - scale = pme->bFEP ? (fep_state < 1 ? 1.0-lambda_lj : lambda_lj) : 1.0; - scale *= lb_scale_factor[grid_index-2]; - #pragma omp parallel for num_threads(pme->nthread) schedule(static) - for (thread = 0; thread < pme->nthread; thread++) + if (bCalcF) { - try + /* interpolate forces for our local atoms */ + where(); + bClearF = (bFirst && PAR(cr)); + scale = pme->bFEP ? (fep_state < 1 ? 1.0-lambda_lj : lambda_lj) : 1.0; + scale *= lb_scale_factor[grid_index-2]; ++ + #pragma omp parallel for num_threads(pme->nthread) schedule(static) + for (thread = 0; thread < pme->nthread; thread++) { -- gather_f_bsplines(pme, grid, bClearF, &pme->atc[0], -- &pme->atc[0].spline[thread], -- scale); ++ try ++ { ++ gather_f_bsplines(pme, grid, bClearF, &pme->atc[0], ++ &pme->atc[0].spline[thread], ++ scale); ++ } ++ GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; } - GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; - } - where(); + - inc_nrnb(nrnb, eNR_GATHERFBSP, - pme->pme_order*pme->pme_order*pme->pme_order*pme->atc[0].n); + where(); + + inc_nrnb(nrnb, eNR_GATHERFBSP, + pme->pme_order*pme->pme_order*pme->pme_order*pme->atc[0].n); + } wallcycle_stop(wcycle, ewcPME_SPREADGATHER); bFirst = FALSE; diff --cc src/gromacs/fileio/readinp.cpp index 6a5f63f40a,0000000000..0f182a718c mode 100644,000000..100644 --- a/src/gromacs/fileio/readinp.cpp +++ b/src/gromacs/fileio/readinp.cpp @@@ -1,549 -1,0 +1,557 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 1991-2000, University of Groningen, The Netherlands. + * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +#include "gmxpre.h" + +#include "readinp.h" + +#include +#include +#include + +#include + +#include "gromacs/fileio/gmxfio.h" +#include "gromacs/fileio/warninp.h" +#include "gromacs/mdtypes/md_enums.h" +#include "gromacs/utility/binaryinformation.h" +#include "gromacs/utility/cstringutil.h" +#include "gromacs/utility/exceptions.h" +#include "gromacs/utility/fatalerror.h" +#include "gromacs/utility/futil.h" +#include "gromacs/utility/programcontext.h" +#include "gromacs/utility/qsort_threadsafe.h" +#include "gromacs/utility/smalloc.h" + +t_inpfile *read_inpfile(const char *fn, int *ninp, + warninp_t wi) +{ + FILE *in; + char buf[STRLEN], lbuf[STRLEN], rbuf[STRLEN], warn_buf[STRLEN]; + char *ptr, *cptr; + t_inpfile *inp = NULL; + int nin, lc, i, j, k; + /* setting cppopts from command-line options would be cooler */ + gmx_bool allow_override = FALSE; + + + if (debug) + { + fprintf(debug, "Reading MDP file %s\n", fn); + } + + in = gmx_ffopen(fn, "r"); + + nin = lc = 0; + do + { + ptr = fgets2(buf, STRLEN-1, in); + lc++; + set_warning_line(wi, fn, lc); + if (ptr) + { ++ // TODO This parsing should be using strip_comment, trim, ++ // strchr, etc. rather than re-inventing wheels. ++ + /* Strip comment */ + if ((cptr = std::strchr(buf, COMMENTSIGN)) != NULL) + { + *cptr = '\0'; + } + /* Strip spaces */ + trim(buf); + + for (j = 0; (buf[j] != '=') && (buf[j] != '\0'); j++) + { + ; + } + if (buf[j] == '\0') + { + if (j > 0) + { + if (debug) + { + fprintf(debug, "No = on line %d in file %s, ignored\n", lc, fn); + } + } + } + else + { + for (i = 0; (i < j); i++) + { + lbuf[i] = buf[i]; + } + lbuf[i] = '\0'; + trim(lbuf); + if (lbuf[0] == '\0') + { + if (debug) + { + fprintf(debug, "Empty left hand side on line %d in file %s, ignored\n", lc, fn); + } + } + else + { + for (i = j+1, k = 0; (buf[i] != '\0'); i++, k++) + { + rbuf[k] = buf[i]; + } + rbuf[k] = '\0'; + trim(rbuf); + if (rbuf[0] == '\0') + { + if (debug) + { + fprintf(debug, "Empty right hand side on line %d in file %s, ignored\n", lc, fn); + } + } + else + { + /* Now finally something sensible */ + int found_index; + + /* first check whether we hit the 'multiple_entries' option */ + if (gmx_strcasecmp_min(eMultentOpt_names[eMultentOptName], lbuf) == 0) + { + /* we now check whether to allow overrides from here or not */ + if (gmx_strcasecmp_min(eMultentOpt_names[eMultentOptNo], rbuf) == 0) + { + allow_override = FALSE; + } + else if (gmx_strcasecmp_min(eMultentOpt_names[eMultentOptLast], rbuf) == 0) + { + allow_override = TRUE; + } + else + { + sprintf(warn_buf, + "Parameter \"%s\" should either be %s or %s\n", + lbuf, + eMultentOpt_names[eMultentOptNo], + eMultentOpt_names[eMultentOptLast]); + warning_error(wi, warn_buf); + } + } + else + { + /* it is a regular option; check for duplicates */ + found_index = search_einp(nin, inp, lbuf); + + if (found_index == -1) + { + /* add a new item */ + srenew(inp, ++nin); + inp[nin-1].inp_count = 1; + inp[nin-1].count = 0; + inp[nin-1].bObsolete = FALSE; + inp[nin-1].bSet = FALSE; + inp[nin-1].name = gmx_strdup(lbuf); + inp[nin-1].value = gmx_strdup(rbuf); + } + else + { + if (!allow_override) + { + sprintf(warn_buf, + "Parameter \"%s\" doubly defined (and multiple assignments not allowed)\n", + lbuf); + warning_error(wi, warn_buf); + } + else + { + /* override */ + if (!inp) + { + gmx_fatal(FARGS, "Internal inconsistency; inp[] base pointer is NULL"); + } + sfree(inp[found_index].value); + inp[found_index].value = gmx_strdup(rbuf); + sprintf(warn_buf, + "Overriding existing parameter \"%s\" with value \"%s\"\n", + lbuf, rbuf); + warning_note(wi, warn_buf); + } + } + } + } + } + } + } + } + while (ptr); + + gmx_ffclose(in); + + if (debug) + { + fprintf(debug, "Done reading MDP file, there were %d entries in there\n", + nin); + } + + *ninp = nin; + + return inp; +} + + + + +static int inp_comp(const void *a, const void *b) +{ + return (reinterpret_cast(a))->count - (reinterpret_cast(b))->count; +} + +static void sort_inp(int ninp, t_inpfile inp[]) +{ + int i, mm; + + mm = -1; + for (i = 0; (i < ninp); i++) + { + mm = std::max(mm, inp[i].count); + } + for (i = 0; (i < ninp); i++) + { + if (inp[i].count == 0) + { + inp[i].count = mm++; + } + } + gmx_qsort(inp, ninp, static_cast(sizeof(inp[0])), inp_comp); +} + +void write_inpfile(const char *fn, int ninp, t_inpfile inp[], gmx_bool bHaltOnUnknown, + warninp_t wi) +{ + FILE *out; + int i; + char warn_buf[STRLEN]; + + sort_inp(ninp, inp); + out = gmx_fio_fopen(fn, "w"); + nice_header(out, fn); + try + { + gmx::BinaryInformationSettings settings; + settings.generatedByHeader(true); + settings.linePrefix(";\t"); + gmx::printBinaryInformation(out, gmx::getProgramContext(), settings); + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + + for (i = 0; (i < ninp); i++) + { + if (inp[i].bSet) + { + if (inp[i].name[0] == ';' || (strlen(inp[i].name) > 2 && inp[i].name[1] == ';')) + { + fprintf(out, "%-24s\n", inp[i].name); + } + else + { + fprintf(out, "%-24s = %s\n", inp[i].name, inp[i].value ? inp[i].value : ""); + } + } + else if (!inp[i].bObsolete) + { + sprintf(warn_buf, "Unknown left-hand '%s' in parameter file\n", + inp[i].name); + if (bHaltOnUnknown) + { + warning_error(wi, warn_buf); + } + else + { + warning(wi, warn_buf); + } + } + } + gmx_fio_fclose(out); + + check_warning_error(wi, FARGS); +} + +void replace_inp_entry(int ninp, t_inpfile *inp, const char *old_entry, const char *new_entry) +{ + int i; + + for (i = 0; (i < ninp); i++) + { + if (gmx_strcasecmp_min(old_entry, inp[i].name) == 0) + { + if (new_entry) + { + fprintf(stderr, "Replacing old mdp entry '%s' by '%s'\n", + inp[i].name, new_entry); + sfree(inp[i].name); + inp[i].name = gmx_strdup(new_entry); + } + else + { + fprintf(stderr, "Ignoring obsolete mdp entry '%s'\n", + inp[i].name); + inp[i].bObsolete = TRUE; + } + } + } +} + +int search_einp(int ninp, const t_inpfile *inp, const char *name) +{ + int i; + + if (inp == NULL) + { + return -1; + } + for (i = 0; i < ninp; i++) + { + if (gmx_strcasecmp_min(name, inp[i].name) == 0) + { + return i; + } + } + return -1; +} + +static int get_einp(int *ninp, t_inpfile **inp, const char *name) +{ + int i; + int notfound = FALSE; + + i = search_einp(*ninp, *inp, name); + if (i == -1) + { + notfound = TRUE; + i = (*ninp)++; + srenew(*inp, (*ninp)); + (*inp)[i].name = gmx_strdup(name); + (*inp)[i].bSet = TRUE; + } + (*inp)[i].count = (*inp)[0].inp_count++; + (*inp)[i].bSet = TRUE; + if (debug) + { + fprintf(debug, "Inp %d = %s\n", (*inp)[i].count, (*inp)[i].name); + } + + /*if (i == (*ninp)-1)*/ + if (notfound) + { + return -1; + } + else + { + return i; + } +} + ++/* Note that sanitizing the trailing part of (*inp)[ii].value was the responsibility of read_inpfile() */ +int get_eint(int *ninp, t_inpfile **inp, const char *name, int def, + warninp_t wi) +{ + char buf[32], *ptr, warn_buf[STRLEN]; + int ii; + int ret; + + ii = get_einp(ninp, inp, name); + + if (ii == -1) + { + sprintf(buf, "%d", def); + (*inp)[(*ninp)-1].value = gmx_strdup(buf); + + return def; + } + else + { + ret = std::strtol((*inp)[ii].value, &ptr, 10); - if (ptr == (*inp)[ii].value) ++ if (*ptr != '\0') + { + sprintf(warn_buf, "Right hand side '%s' for parameter '%s' in parameter file is not an integer value\n", (*inp)[ii].value, (*inp)[ii].name); + warning_error(wi, warn_buf); + } + + return ret; + } +} + ++/* Note that sanitizing the trailing part of (*inp)[ii].value was the responsibility of read_inpfile() */ +gmx_int64_t get_eint64(int *ninp, t_inpfile **inp, + const char *name, gmx_int64_t def, + warninp_t wi) +{ + char buf[32], *ptr, warn_buf[STRLEN]; + int ii; + gmx_int64_t ret; + + ii = get_einp(ninp, inp, name); + + if (ii == -1) + { + sprintf(buf, "%" GMX_PRId64, def); + (*inp)[(*ninp)-1].value = gmx_strdup(buf); + + return def; + } + else + { + ret = str_to_int64_t((*inp)[ii].value, &ptr); - if (ptr == (*inp)[ii].value) ++ if (*ptr != '\0') + { + sprintf(warn_buf, "Right hand side '%s' for parameter '%s' in parameter file is not an integer value\n", (*inp)[ii].value, (*inp)[ii].name); + warning_error(wi, warn_buf); + } + + return ret; + } +} + ++/* Note that sanitizing the trailing part of (*inp)[ii].value was the responsibility of read_inpfile() */ +double get_ereal(int *ninp, t_inpfile **inp, const char *name, double def, + warninp_t wi) +{ + char buf[32], *ptr, warn_buf[STRLEN]; + int ii; + double ret; + + ii = get_einp(ninp, inp, name); + + if (ii == -1) + { + sprintf(buf, "%g", def); + (*inp)[(*ninp)-1].value = gmx_strdup(buf); + + return def; + } + else + { + ret = strtod((*inp)[ii].value, &ptr); - if (ptr == (*inp)[ii].value) ++ if (*ptr != '\0') + { + sprintf(warn_buf, "Right hand side '%s' for parameter '%s' in parameter file is not a real value\n", (*inp)[ii].value, (*inp)[ii].name); + warning_error(wi, warn_buf); + } + + return ret; + } +} + ++/* Note that sanitizing the trailing part of (*inp)[ii].value was the responsibility of read_inpfile() */ +const char *get_estr(int *ninp, t_inpfile **inp, const char *name, const char *def) +{ + char buf[32]; + int ii; + + ii = get_einp(ninp, inp, name); + + if (ii == -1) + { + if (def) + { + sprintf(buf, "%s", def); + (*inp)[(*ninp)-1].value = gmx_strdup(buf); + } + else + { + (*inp)[(*ninp)-1].value = NULL; + } + + return def; + } + else + { + return (*inp)[ii].value; + } +} + ++/* Note that sanitizing the trailing part of (*inp)[ii].value was the responsibility of read_inpfile() */ +int get_eeenum(int *ninp, t_inpfile **inp, const char *name, const char **defs, + warninp_t wi) +{ + int ii, i, j; + int n = 0; + char buf[STRLEN]; + + ii = get_einp(ninp, inp, name); + + if (ii == -1) + { + (*inp)[(*ninp)-1].value = gmx_strdup(defs[0]); + + return 0; + } + + for (i = 0; (defs[i] != NULL); i++) + { + if (gmx_strcasecmp_min(defs[i], (*inp)[ii].value) == 0) + { + break; + } + } + + if (defs[i] == NULL) + { + n += sprintf(buf, "Invalid enum '%s' for variable %s, using '%s'\n", + (*inp)[ii].value, name, defs[0]); + n += sprintf(buf+n, "Next time use one of:"); + j = 0; + while (defs[j]) + { + n += sprintf(buf+n, " '%s'", defs[j]); + j++; + } + if (wi != NULL) + { + warning_error(wi, buf); + } + else + { + fprintf(stderr, "%s\n", buf); + } + + (*inp)[ii].value = gmx_strdup(defs[0]); + + return 0; + } + + return i; +} + +int get_eenum(int *ninp, t_inpfile **inp, const char *name, const char **defs) +{ + return get_eeenum(ninp, inp, name, defs, NULL); +} diff --cc src/gromacs/fileio/tests/CMakeLists.txt index 8f83ed1cca,b0f62731b6..d800b19cdf --- a/src/gromacs/fileio/tests/CMakeLists.txt +++ b/src/gromacs/fileio/tests/CMakeLists.txt @@@ -1,7 -1,7 +1,7 @@@ # # This file is part of the GROMACS molecular simulation package. # - # Copyright (c) 2013,2014,2015, by the GROMACS development team, led by -# Copyright (c) 2013,2014, by the GROMACS development team, led by ++# Copyright (c) 2013,2014,2015,2016, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. @@@ -32,10 -32,7 +32,11 @@@ # To help us fund GROMACS development, we humbly ask that you cite # the research papers on the package. Check out http://www.gromacs.org. -if(GMX_USE_TNG) - gmx_add_unit_test(FileIOTests fileio-test - tngio.cpp) +set(test_sources + confio.cpp ++ readinp.cpp + ) +if (GMX_USE_TNG) + list(APPEND test_sources tngio.cpp) endif() +gmx_add_unit_test(FileIOTests fileio-test ${test_sources}) diff --cc src/gromacs/fileio/tests/readinp.cpp index 0000000000,0000000000..506a3dbd1d new file mode 100644 --- /dev/null +++ b/src/gromacs/fileio/tests/readinp.cpp @@@ -1,0 -1,0 +1,149 @@@ ++/* ++ * This file is part of the GROMACS molecular simulation package. ++ * ++ * Copyright (c) 2016, by the GROMACS development team, led by ++ * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, ++ * and including many others, as listed in the AUTHORS file in the ++ * top-level source directory and at http://www.gromacs.org. ++ * ++ * GROMACS is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public License ++ * as published by the Free Software Foundation; either version 2.1 ++ * of the License, or (at your option) any later version. ++ * ++ * GROMACS is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with GROMACS; if not, see ++ * http://www.gnu.org/licenses, or write to the Free Software Foundation, ++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * If you want to redistribute modifications to GROMACS, please ++ * consider that scientific software is very special. Version ++ * control is crucial - bugs must be traceable. We will be happy to ++ * consider code for inclusion in the official distribution, but ++ * derived work must not be called official GROMACS. Details are found ++ * in the README & COPYING files - if they are missing, get the ++ * official version at http://www.gromacs.org. ++ * ++ * To help us fund GROMACS development, we humbly ask that you cite ++ * the research papers on the package. Check out http://www.gromacs.org. ++ */ ++/*! \internal \file ++ * \brief ++ * Tests utilities for routines that parse fields e.g. from grompp input ++ * ++ * \author Mark Abraham ++ */ ++#include "gmxpre.h" ++ ++#include "gromacs/fileio/readinp.h" ++ ++#include ++ ++#include "gromacs/fileio/warninp.h" ++#include "gromacs/utility/scoped_cptr.h" ++#include "gromacs/utility/smalloc.h" ++ ++namespace gmx ++{ ++namespace testing ++{ ++ ++class ReadTest : public ::testing::Test ++{ ++ public: ++ ReadTest() : numInputs_(1), ++ inputField_(0), ++ inpGuard_(), ++ wi_(), ++ wiGuard_() ++ { ++ snew(inputField_, numInputs_); ++ inpGuard_.reset(inputField_); ++ ++ inputField_[0].count = 0; ++ inputField_[0].bObsolete = FALSE; ++ inputField_[0].bSet = FALSE; ++ inputField_[0].name = (char *) "test"; ++ inputField_[0].inp_count = 0; ++ ++ wi_ = init_warning(FALSE, 0); ++ wiGuard_.reset(wi_); ++ } ++ ++ int numInputs_; ++ t_inpfile *inputField_; ++ gmx::scoped_cptr inpGuard_; ++ warninp_t wi_; ++ gmx::scoped_cptr wiGuard_; ++}; ++ ++TEST_F(ReadTest, get_eint_ReadsInteger) ++{ ++ inputField_[0].value = (char *) "1"; ++ ASSERT_EQ(1, get_eint(&numInputs_, &inputField_, "test", 2, wi_)); ++ ASSERT_FALSE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_eint_WarnsAboutFloat) ++{ ++ inputField_[0].value = (char *) "0.8"; ++ get_eint(&numInputs_, &inputField_, "test", 2, wi_); ++ ASSERT_TRUE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_eint_WarnsAboutString) ++{ ++ inputField_[0].value = (char *) "hello"; ++ get_eint(&numInputs_, &inputField_, "test", 2, wi_); ++ ASSERT_TRUE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_eint64_ReadsInteger) ++{ ++ inputField_[0].value = (char *) "1"; ++ ASSERT_EQ(1, get_eint64(&numInputs_, &inputField_, "test", 2, wi_)); ++ ASSERT_FALSE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_eint64_WarnsAboutFloat) ++{ ++ inputField_[0].value = (char *) "0.8"; ++ get_eint64(&numInputs_, &inputField_, "test", 2, wi_); ++ ASSERT_TRUE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_eint64_WarnsAboutString) ++{ ++ inputField_[0].value = (char *) "hello"; ++ get_eint64(&numInputs_, &inputField_, "test", 2, wi_); ++ ASSERT_TRUE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_ereal_ReadsInteger) ++{ ++ inputField_[0].value = (char *) "1"; ++ ASSERT_EQ(1, get_ereal(&numInputs_, &inputField_, "test", 2, wi_)); ++ ASSERT_FALSE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_ereal_ReadsFloat) ++{ ++ inputField_[0].value = (char *) "0.8"; ++ ASSERT_EQ(0.8, get_ereal(&numInputs_, &inputField_, "test", 2, wi_)); ++ ASSERT_FALSE(warning_errors_exist(wi_)); ++} ++ ++TEST_F(ReadTest, get_ereal_WarnsAboutString) ++{ ++ inputField_[0].value = (char *) "hello"; ++ get_ereal(&numInputs_, &inputField_, "test", 2, wi_); ++ ASSERT_TRUE(warning_errors_exist(wi_)); ++} ++ ++} // namespace ++} // namespace diff --cc src/gromacs/fileio/warninp.cpp index c247852c29,0000000000..d6231cfa31 mode 100644,000000..100644 --- a/src/gromacs/fileio/warninp.cpp +++ b/src/gromacs/fileio/warninp.cpp @@@ -1,215 -1,0 +1,225 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 1991-2000, University of Groningen, The Netherlands. + * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +#include "gmxpre.h" + +#include "warninp.h" + +#include + +#include "gromacs/utility/cstringutil.h" +#include "gromacs/utility/fatalerror.h" +#include "gromacs/utility/smalloc.h" + +typedef struct warninp { + gmx_bool bAllowWarnings; + int nwarn_note; + int nwarn_warn; + int nwarn_error; + int maxwarn; + int lineno; + char filenm[256]; +} t_warninp; + +warninp_t init_warning(gmx_bool bAllowWarnings, int maxwarning) +{ + warninp_t wi; + + snew(wi, 1); + + wi->bAllowWarnings = bAllowWarnings; + wi->maxwarn = maxwarning; + wi->nwarn_note = 0; + wi->nwarn_warn = 0; + wi->nwarn_error = 0; + strcpy(wi->filenm, "unknown"); + wi->lineno = 0; + + return wi; +} + +void set_warning_line(warninp_t wi, const char *s, int line) +{ + if (s != NULL) + { + std::strcpy(wi->filenm, s); + } + wi->lineno = line; +} + +int get_warning_line(warninp_t wi) +{ + return wi->lineno; +} + +const char *get_warning_file(warninp_t wi) +{ + return wi->filenm; +} + +static void low_warning(warninp_t wi, const char *wtype, int n, const char *s) +{ +#define indent 2 + char *temp, *temp2; + int i; + + if (s == NULL) + { + s = "Empty error message."; + } + snew(temp, std::strlen(s)+indent+1); + for (i = 0; i < indent; i++) + { + temp[i] = ' '; + } + temp[indent] = '\0'; + std::strcat(temp, s); + temp2 = wrap_lines(temp, 78-indent, indent, FALSE); + if (std::strlen(wi->filenm) > 0) + { + if (wi->lineno != -1) + { + fprintf(stderr, "\n%s %d [file %s, line %d]:\n%s\n\n", + wtype, n, wi->filenm, wi->lineno, temp2); + } + else + { + fprintf(stderr, "\n%s %d [file %s]:\n%s\n\n", + wtype, n, wi->filenm, temp2); + } + } + else + { + fprintf(stderr, "\n%s %d:\n%s\n\n", wtype, n, temp2); + } + sfree(temp); + sfree(temp2); +} + +void warning(warninp_t wi, const char *s) +{ + if (wi->bAllowWarnings) + { + wi->nwarn_warn++; + low_warning(wi, "WARNING", wi->nwarn_warn, s); + } + else + { + warning_error(wi, s); + } +} + +void warning_note(warninp_t wi, const char *s) +{ + wi->nwarn_note++; + low_warning(wi, "NOTE", wi->nwarn_note, s); +} + +void warning_error(warninp_t wi, const char *s) +{ + wi->nwarn_error++; + low_warning(wi, "ERROR", wi->nwarn_error, s); +} + +static void print_warn_count(const char *type, int n) +{ + if (n > 0) + { + fprintf(stderr, "\nThere %s %d %s%s\n", + (n == 1) ? "was" : "were", n, type, (n == 1) ? "" : "s"); + } +} + +void check_warning_error(warninp_t wi, int f_errno, const char *file, int line) +{ + if (wi->nwarn_error > 0) + { + print_warn_count("note", wi->nwarn_note); + print_warn_count("warning", wi->nwarn_warn); + + gmx_fatal(f_errno, file, line, "There %s %d error%s in input file(s)", + (wi->nwarn_error == 1) ? "was" : "were", wi->nwarn_error, + (wi->nwarn_error == 1) ? "" : "s"); + } +} + ++gmx_bool warning_errors_exist(warninp_t wi) ++{ ++ return (wi->nwarn_error > 0); ++} ++ +void done_warning(warninp_t wi, int f_errno, const char *file, int line) +{ + print_warn_count("note", wi->nwarn_note); + print_warn_count("warning", wi->nwarn_warn); + + check_warning_error(wi, f_errno, file, line); + + if (wi->maxwarn >= 0 && wi->nwarn_warn > wi->maxwarn) + { + gmx_fatal(f_errno, file, line, + "Too many warnings (%d).\n" + "If you are sure all warnings are harmless, use the -maxwarn option.", + wi->nwarn_warn); + } + ++ free_warning(wi); ++} ++ ++void free_warning(warninp_t wi) ++{ + sfree(wi); +} + +void _too_few(warninp_t wi, const char *fn, int line) +{ + char buf[STRLEN]; + + sprintf(buf, + "Too few parameters on line (source file %s, line %d)", + fn, line); + warning(wi, buf); +} + +void _incorrect_n_param(warninp_t wi, const char *fn, int line) +{ + char buf[STRLEN]; + + sprintf(buf, + "Incorrect number of parameters on line (source file %s, line %d)", + fn, line); + warning(wi, buf); +} diff --cc src/gromacs/fileio/warninp.h index 1324e4996c,0000000000..d8efa10e7c mode 100644,000000..100644 --- a/src/gromacs/fileio/warninp.h +++ b/src/gromacs/fileio/warninp.h @@@ -1,123 -1,0 +1,130 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 1991-2000, University of Groningen, The Netherlands. + * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2010,2014,2015, by the GROMACS development team, led by ++ * Copyright (c) 2010,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +#ifndef GMX_FILEIO_WARNINP_H +#define GMX_FILEIO_WARNINP_H + +#include "gromacs/utility/basedefinitions.h" + +/* Abstract type for warning bookkeeping */ +typedef struct warninp *warninp_t; + + +warninp_t +init_warning(gmx_bool bAllowWarnings, int maxwarning); +/* Initialize the warning data structure. + * If bAllowWarnings=FALSE, all warnings (calls to warning()) will be + * transformed into errors, calls to warning_note still produce notes. + * maxwarning determines the maximum number of warnings that are allowed + * for proceeding. When this number is exceeded check_warning_error + * and done_warning will generate a fatal error. + * bAllowWarnings=TRUE should only be used by programs that have + * a -maxwarn command line option. + */ + +void +set_warning_line(warninp_t wi, const char *fn, int line); +/* Set filename and linenumber for the warning */ + +int +get_warning_line(warninp_t wi); +/* Get linenumber for the warning */ + + +const char * +get_warning_file(warninp_t wi); +/* Get filename for the warning */ + +void +warning(warninp_t wi, const char *s); +/* Issue a warning, with the string s. If s == NULL, then warn_buf + * will be printed instead. The file and line set by set_warning_line + * are printed, nwarn_warn (local) is incremented. + * A fatal error will be generated after processing the input + * when nwarn_warn is larger than maxwarning passed to init_warning. + * So warning should only be called for issues that should be resolved, + * otherwise warning_note should be called. + */ + +void +warning_note(warninp_t wi, const char *s); +/* Issue a note, with the string s. If s == NULL, then warn_buf + * will be printed instead. The file and line set by set_warning_line + * are printed, nwarn_note (local) is incremented. + * This is for issues which could be a problem for some systems, + * but 100% ok for other systems. + */ + +void +warning_error(warninp_t wi, const char *s); +/* Issue an error, with the string s. If s == NULL, then warn_buf + * will be printed instead. The file and line set by set_warning_line + * are printed, nwarn_error (local) is incremented. + */ + ++gmx_bool warning_errors_exist(warninp_t wi); ++/* Return whether any error-level warnings were issued to wi. */ ++ +void +check_warning_error(warninp_t wi, int f_errno, const char *file, int line); +/* When warning_error has been called at least once gmx_fatal is called, + * otherwise does nothing. + */ + +void +done_warning(warninp_t wi, int f_errno, const char *file, int line); +/* Should be called when finished processing the input file. + * Prints the number of notes and warnings + * and generates a fatal error when errors were found or too many + * warnings were generatesd. + * Frees the data structure pointed to by wi. + */ + ++void ++free_warning(warninp_t wi); ++/* Frees the data structure pointed to by wi. */ ++ +void +_too_few(warninp_t wi, const char *fn, int line); +#define too_few(wi) _too_few(wi, __FILE__, __LINE__) +/* Issue a warning stating 'Too few parameters' */ + +void +_incorrect_n_param(warninp_t wi, const char *fn, int line); +#define incorrect_n_param(wi) _incorrect_n_param(wi, __FILE__, __LINE__) +/* Issue a warning stating 'Incorrect number of parameters' */ + +#endif diff --cc src/gromacs/gmxlib/CMakeLists.txt index 22ac02f65f,9a9df8b1a1..50d052de34 --- a/src/gromacs/gmxlib/CMakeLists.txt +++ b/src/gromacs/gmxlib/CMakeLists.txt @@@ -36,6 -38,20 +36,10 @@@ add_subdirectory(nonbonded # The nonbonded directory contains subdirectories that are only # conditionally built, so we cannot use a GLOB_RECURSE here. -file(GLOB GMXLIB_SOURCES *.c *.cpp) - -# gpu utils + cuda tools module -if(GMX_GPU) - if(NOT GMX_USE_OPENCL) - add_subdirectory(cuda_tools) - else() - add_subdirectory(ocl_tools) - endif() -endif() -add_subdirectory(gpu_utils) +file(GLOB GMXLIB_SOURCES *.cpp) set(GMXLIB_SOURCES ${GMXLIB_SOURCES} ${NONBONDED_SOURCES} PARENT_SCOPE) + -if(BUILD_TESTING) - add_subdirectory(tests) -endif() ++# if(BUILD_TESTING) ++# add_subdirectory(tests) ++# endif() diff --cc src/gromacs/gmxlib/tests/CMakeLists.txt index 0000000000,969910819b..66ee98c31e mode 000000,100644..100644 --- a/src/gromacs/gmxlib/tests/CMakeLists.txt +++ b/src/gromacs/gmxlib/tests/CMakeLists.txt @@@ -1,0 -1,37 +1,36 @@@ + # + # This file is part of the GROMACS molecular simulation package. + # + # Copyright (c) 2016, by the GROMACS development team, led by + # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + # and including many others, as listed in the AUTHORS file in the + # top-level source directory and at http://www.gromacs.org. + # + # GROMACS is free software; you can redistribute it and/or + # modify it under the terms of the GNU Lesser General Public License + # as published by the Free Software Foundation; either version 2.1 + # of the License, or (at your option) any later version. + # + # GROMACS is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + # Lesser General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public + # License along with GROMACS; if not, see + # http://www.gnu.org/licenses, or write to the Free Software Foundation, + # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + # + # If you want to redistribute modifications to GROMACS, please + # consider that scientific software is very special. Version + # control is crucial - bugs must be traceable. We will be happy to + # consider code for inclusion in the official distribution, but + # derived work must not be called official GROMACS. Details are found + # in the README & COPYING files - if they are missing, get the + # official version at http://www.gromacs.org. + # + # To help us fund GROMACS development, we humbly ask that you cite + # the research papers on the package. Check out http://www.gromacs.org. + + gmx_add_unit_test(GmxlibTests gmxlib-test - readinp.cpp + ) diff --cc src/gromacs/gmxpreprocess/toppush.cpp index 35d80c6cb9,0000000000..d7ff189ea0 mode 100644,000000..100644 --- a/src/gromacs/gmxpreprocess/toppush.cpp +++ b/src/gromacs/gmxpreprocess/toppush.cpp @@@ -1,2738 -1,0 +1,2750 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 1991-2000, University of Groningen, The Netherlands. + * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +#include "gmxpre.h" + +#include "toppush.h" + +#include +#include + +#include + +#include + +#include "gromacs/fileio/warninp.h" +#include "gromacs/gmxpreprocess/gpp_atomtype.h" +#include "gromacs/gmxpreprocess/gpp_bond_atomtype.h" +#include "gromacs/gmxpreprocess/notset.h" +#include "gromacs/gmxpreprocess/readir.h" +#include "gromacs/gmxpreprocess/topdirs.h" +#include "gromacs/gmxpreprocess/toputil.h" +#include "gromacs/mdtypes/md_enums.h" +#include "gromacs/topology/ifunc.h" +#include "gromacs/topology/symtab.h" +#include "gromacs/utility/cstringutil.h" +#include "gromacs/utility/fatalerror.h" +#include "gromacs/utility/gmxassert.h" +#include "gromacs/utility/smalloc.h" + +void generate_nbparams(int comb, int ftype, t_params *plist, gpp_atomtype_t atype, + warninp_t wi) +{ + int i, j, k = -1, nf; + int nr, nrfp; + real c, bi, bj, ci, cj, ci0, ci1, ci2, cj0, cj1, cj2; + char errbuf[256]; + + /* Lean mean shortcuts */ + nr = get_atomtype_ntypes(atype); + nrfp = NRFP(ftype); + snew(plist->param, nr*nr); + plist->nr = nr*nr; + + /* Fill the matrix with force parameters */ + switch (ftype) + { + case F_LJ: + switch (comb) + { + case eCOMB_GEOMETRIC: + /* Gromos rules */ + for (i = k = 0; (i < nr); i++) + { + for (j = 0; (j < nr); j++, k++) + { + for (nf = 0; (nf < nrfp); nf++) + { + ci = get_atomtype_nbparam(i, nf, atype); + cj = get_atomtype_nbparam(j, nf, atype); + c = std::sqrt(ci * cj); + plist->param[k].c[nf] = c; + } + } + } + break; + + case eCOMB_ARITHMETIC: + /* c0 and c1 are sigma and epsilon */ + for (i = k = 0; (i < nr); i++) + { + for (j = 0; (j < nr); j++, k++) + { + ci0 = get_atomtype_nbparam(i, 0, atype); + cj0 = get_atomtype_nbparam(j, 0, atype); + ci1 = get_atomtype_nbparam(i, 1, atype); + cj1 = get_atomtype_nbparam(j, 1, atype); + plist->param[k].c[0] = (fabs(ci0) + fabs(cj0))*0.5; + /* Negative sigma signals that c6 should be set to zero later, + * so we need to propagate that through the combination rules. + */ + if (ci0 < 0 || cj0 < 0) + { + plist->param[k].c[0] *= -1; + } + plist->param[k].c[1] = std::sqrt(ci1*cj1); + } + } + + break; + case eCOMB_GEOM_SIG_EPS: + /* c0 and c1 are sigma and epsilon */ + for (i = k = 0; (i < nr); i++) + { + for (j = 0; (j < nr); j++, k++) + { + ci0 = get_atomtype_nbparam(i, 0, atype); + cj0 = get_atomtype_nbparam(j, 0, atype); + ci1 = get_atomtype_nbparam(i, 1, atype); + cj1 = get_atomtype_nbparam(j, 1, atype); + plist->param[k].c[0] = std::sqrt(fabs(ci0*cj0)); + /* Negative sigma signals that c6 should be set to zero later, + * so we need to propagate that through the combination rules. + */ + if (ci0 < 0 || cj0 < 0) + { + plist->param[k].c[0] *= -1; + } + plist->param[k].c[1] = std::sqrt(ci1*cj1); + } + } + + break; + default: + gmx_fatal(FARGS, "No such combination rule %d", comb); + } + if (plist->nr != k) + { + gmx_incons("Topology processing, generate nb parameters"); + } + break; + + case F_BHAM: + /* Buckingham rules */ + for (i = k = 0; (i < nr); i++) + { + for (j = 0; (j < nr); j++, k++) + { + ci0 = get_atomtype_nbparam(i, 0, atype); + cj0 = get_atomtype_nbparam(j, 0, atype); + ci2 = get_atomtype_nbparam(i, 2, atype); + cj2 = get_atomtype_nbparam(j, 2, atype); + bi = get_atomtype_nbparam(i, 1, atype); + bj = get_atomtype_nbparam(j, 1, atype); + plist->param[k].c[0] = std::sqrt(ci0 * cj0); + if ((bi == 0) || (bj == 0)) + { + plist->param[k].c[1] = 0; + } + else + { + plist->param[k].c[1] = 2.0/(1/bi+1/bj); + } + plist->param[k].c[2] = std::sqrt(ci2 * cj2); + } + } + + break; + default: + sprintf(errbuf, "Invalid nonbonded type %s", + interaction_function[ftype].longname); + warning_error(wi, errbuf); + } +} + +static void realloc_nb_params(gpp_atomtype_t at, + t_nbparam ***nbparam, t_nbparam ***pair) +{ + /* Add space in the non-bonded parameters matrix */ + int atnr = get_atomtype_ntypes(at); + srenew(*nbparam, atnr); + snew((*nbparam)[atnr-1], atnr); + if (pair) + { + srenew(*pair, atnr); + snew((*pair)[atnr-1], atnr); + } +} + +static void copy_B_from_A(int ftype, double *c) +{ + int nrfpA, nrfpB, i; + + nrfpA = NRFPA(ftype); + nrfpB = NRFPB(ftype); + + /* Copy the B parameters from the first nrfpB A parameters */ + for (i = 0; (i < nrfpB); i++) + { + c[nrfpA+i] = c[i]; + } +} + +void push_at (t_symtab *symtab, gpp_atomtype_t at, t_bond_atomtype bat, + char *line, int nb_funct, + t_nbparam ***nbparam, t_nbparam ***pair, + warninp_t wi) +{ + typedef struct { + const char *entry; + int ptype; + } t_xlate; + t_xlate xl[eptNR] = { + { "A", eptAtom }, + { "N", eptNucleus }, + { "S", eptShell }, + { "B", eptBond }, + { "V", eptVSite }, + }; + + int nr, i, nfields, j, pt, nfp0 = -1; + int batype_nr, nread; + char type[STRLEN], btype[STRLEN], ptype[STRLEN]; + double m, q; + double c[MAXFORCEPARAM]; + double radius, vol, surftens, gb_radius, S_hct; + char tmpfield[12][100]; /* Max 12 fields of width 100 */ + char errbuf[256]; + t_atom *atom; + t_param *param; + int atomnr; + gmx_bool have_atomic_number; + gmx_bool have_bonded_type; + + snew(atom, 1); + snew(param, 1); + + /* First assign input line to temporary array */ + nfields = sscanf(line, "%s%s%s%s%s%s%s%s%s%s%s%s", + tmpfield[0], tmpfield[1], tmpfield[2], tmpfield[3], tmpfield[4], tmpfield[5], + tmpfield[6], tmpfield[7], tmpfield[8], tmpfield[9], tmpfield[10], tmpfield[11]); + + /* Comments on optional fields in the atomtypes section: + * + * The force field format is getting a bit old. For OPLS-AA we needed + * to add a special bonded atomtype, and for Gerrit Groenhofs QM/MM stuff + * we also needed the atomic numbers. + * To avoid making all old or user-generated force fields unusable we + * have introduced both these quantities as optional columns, and do some + * acrobatics to check whether they are present or not. + * This will all look much nicer when we switch to XML... sigh. + * + * Field 0 (mandatory) is the nonbonded type name. (string) + * Field 1 (optional) is the bonded type (string) + * Field 2 (optional) is the atomic number (int) + * Field 3 (mandatory) is the mass (numerical) + * Field 4 (mandatory) is the charge (numerical) + * Field 5 (mandatory) is the particle type (single character) + * This is followed by a number of nonbonded parameters. + * + * The safest way to identify the format is the particle type field. + * + * So, here is what we do: + * + * A. Read in the first six fields as strings + * B. If field 3 (starting from 0) is a single char, we have neither + * bonded_type or atomic numbers. + * C. If field 5 is a single char we have both. + * D. If field 4 is a single char we check field 1. If this begins with + * an alphabetical character we have bonded types, otherwise atomic numbers. + */ + + if (nfields < 6) + { + too_few(wi); + return; + } + + if ( (strlen(tmpfield[5]) == 1) && isalpha(tmpfield[5][0]) ) + { + have_bonded_type = TRUE; + have_atomic_number = TRUE; + } + else if ( (strlen(tmpfield[3]) == 1) && isalpha(tmpfield[3][0]) ) + { + have_bonded_type = FALSE; + have_atomic_number = FALSE; + } + else + { + have_bonded_type = ( isalpha(tmpfield[1][0]) != 0 ); + have_atomic_number = !have_bonded_type; + } + + /* optional fields */ + surftens = -1; + vol = -1; + radius = -1; + gb_radius = -1; + atomnr = -1; + S_hct = -1; + + switch (nb_funct) + { + + case F_LJ: + nfp0 = 2; + + if (have_atomic_number) + { + if (have_bonded_type) + { + nread = sscanf(line, "%s%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf", + type, btype, &atomnr, &m, &q, ptype, &c[0], &c[1], + &radius, &vol, &surftens, &gb_radius); + if (nread < 8) + { + too_few(wi); + return; + } + } + else + { + /* have_atomic_number && !have_bonded_type */ + nread = sscanf(line, "%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf", + type, &atomnr, &m, &q, ptype, &c[0], &c[1], + &radius, &vol, &surftens, &gb_radius); + if (nread < 7) + { + too_few(wi); + return; + } + } + } + else + { + if (have_bonded_type) + { + /* !have_atomic_number && have_bonded_type */ + nread = sscanf(line, "%s%s%lf%lf%s%lf%lf%lf%lf%lf%lf", + type, btype, &m, &q, ptype, &c[0], &c[1], + &radius, &vol, &surftens, &gb_radius); + if (nread < 7) + { + too_few(wi); + return; + } + } + else + { + /* !have_atomic_number && !have_bonded_type */ + nread = sscanf(line, "%s%lf%lf%s%lf%lf%lf%lf%lf%lf", + type, &m, &q, ptype, &c[0], &c[1], + &radius, &vol, &surftens, &gb_radius); + if (nread < 6) + { + too_few(wi); + return; + } + } + } + + if (!have_bonded_type) + { + strcpy(btype, type); + } + + if (!have_atomic_number) + { + atomnr = -1; + } + + break; + + case F_BHAM: + nfp0 = 3; + + if (have_atomic_number) + { + if (have_bonded_type) + { + nread = sscanf(line, "%s%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf%lf", + type, btype, &atomnr, &m, &q, ptype, &c[0], &c[1], &c[2], + &radius, &vol, &surftens, &gb_radius); + if (nread < 9) + { + too_few(wi); + return; + } + } + else + { + /* have_atomic_number && !have_bonded_type */ + nread = sscanf(line, "%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf%lf", + type, &atomnr, &m, &q, ptype, &c[0], &c[1], &c[2], + &radius, &vol, &surftens, &gb_radius); + if (nread < 8) + { + too_few(wi); + return; + } + } + } + else + { + if (have_bonded_type) + { + /* !have_atomic_number && have_bonded_type */ + nread = sscanf(line, "%s%s%lf%lf%s%lf%lf%lf%lf%lf%lf%lf", + type, btype, &m, &q, ptype, &c[0], &c[1], &c[2], + &radius, &vol, &surftens, &gb_radius); + if (nread < 8) + { + too_few(wi); + return; + } + } + else + { + /* !have_atomic_number && !have_bonded_type */ + nread = sscanf(line, "%s%lf%lf%s%lf%lf%lf%lf%lf%lf%lf", + type, &m, &q, ptype, &c[0], &c[1], &c[2], + &radius, &vol, &surftens, &gb_radius); + if (nread < 7) + { + too_few(wi); + return; + } + } + } + + if (!have_bonded_type) + { + strcpy(btype, type); + } + + if (!have_atomic_number) + { + atomnr = -1; + } + + break; + + default: + gmx_fatal(FARGS, "Invalid function type %d in push_at %s %d", nb_funct, + __FILE__, __LINE__); + } + for (j = nfp0; (j < MAXFORCEPARAM); j++) + { + c[j] = 0.0; + } + + if (strlen(type) == 1 && isdigit(type[0])) + { + gmx_fatal(FARGS, "Atom type names can't be single digits."); + } + + if (strlen(btype) == 1 && isdigit(btype[0])) + { + gmx_fatal(FARGS, "Bond atom type names can't be single digits."); + } + + /* Hack to read old topologies */ + if (gmx_strcasecmp(ptype, "D") == 0) + { + sprintf(ptype, "V"); + } + for (j = 0; (j < eptNR); j++) + { + if (gmx_strcasecmp(ptype, xl[j].entry) == 0) + { + break; + } + } + if (j == eptNR) + { + gmx_fatal(FARGS, "Invalid particle type %s on line %s", + ptype, line); + } + /* cppcheck-suppress arrayIndexOutOfBounds #6329 */ + pt = xl[j].ptype; + if (debug) + { + fprintf(debug, "ptype: %s\n", ptype_str[pt]); + } + + atom->q = q; + atom->m = m; + atom->ptype = pt; + for (i = 0; (i < MAXFORCEPARAM); i++) + { + param->c[i] = c[i]; + } + + if ((batype_nr = get_bond_atomtype_type(btype, bat)) == NOTSET) + { + add_bond_atomtype(bat, symtab, btype); + } + batype_nr = get_bond_atomtype_type(btype, bat); + + if ((nr = get_atomtype_type(type, at)) != NOTSET) + { + sprintf(errbuf, "Overriding atomtype %s", type); + warning(wi, errbuf); + if ((nr = set_atomtype(nr, at, symtab, atom, type, param, batype_nr, + radius, vol, surftens, atomnr, gb_radius, S_hct)) == NOTSET) + { + gmx_fatal(FARGS, "Replacing atomtype %s failed", type); + } + } + else if ((add_atomtype(at, symtab, atom, type, param, + batype_nr, radius, vol, + surftens, atomnr, gb_radius, S_hct)) == NOTSET) + { + gmx_fatal(FARGS, "Adding atomtype %s failed", type); + } + else + { + /* Add space in the non-bonded parameters matrix */ + realloc_nb_params(at, nbparam, pair); + } + sfree(atom); + sfree(param); +} + +static void push_bondtype(t_params * bt, + t_param * b, + int nral, + int ftype, + gmx_bool bAllowRepeat, + char * line, + warninp_t wi) +{ + int i, j; + gmx_bool bTest, bFound, bCont, bId; + int nr = bt->nr; + int nrfp = NRFP(ftype); + char errbuf[256]; + + /* If bAllowRepeat is TRUE, we allow multiple entries as long as they + are on directly _adjacent_ lines. + */ + + /* First check if our atomtypes are _identical_ (not reversed) to the previous + entry. If they are not identical we search for earlier duplicates. If they are + we can skip it, since we already searched for the first line + in this group. + */ + + bFound = FALSE; + bCont = FALSE; + + if (bAllowRepeat && nr > 1) + { + for (j = 0, bCont = TRUE; (j < nral); j++) + { + bCont = bCont && (b->a[j] == bt->param[nr-2].a[j]); + } + } + + /* Search for earlier duplicates if this entry was not a continuation + from the previous line. + */ + if (!bCont) + { + bFound = FALSE; + for (i = 0; (i < nr); i++) + { + bTest = TRUE; + for (j = 0; (j < nral); j++) + { + bTest = (bTest && (b->a[j] == bt->param[i].a[j])); + } + if (!bTest) + { + bTest = TRUE; + for (j = 0; (j < nral); j++) + { + bTest = (bTest && (b->a[nral-1-j] == bt->param[i].a[j])); + } + } + if (bTest) + { + if (!bFound) + { + bId = TRUE; + for (j = 0; (j < nrfp); j++) + { + bId = bId && (bt->param[i].c[j] == b->c[j]); + } + if (!bId) + { + sprintf(errbuf, "Overriding %s parameters.%s", + interaction_function[ftype].longname, + (ftype == F_PDIHS) ? + "\nUse dihedraltype 9 to allow several multiplicity terms. Only consecutive lines are combined. Non-consective lines overwrite each other." + : ""); + warning(wi, errbuf); + fprintf(stderr, " old: "); + for (j = 0; (j < nrfp); j++) + { + fprintf(stderr, " %g", bt->param[i].c[j]); + } + fprintf(stderr, " \n new: %s\n\n", line); + } + } + /* Overwrite it! */ + for (j = 0; (j < nrfp); j++) + { + bt->param[i].c[j] = b->c[j]; + } + bFound = TRUE; + } + } + } + if (!bFound) + { + /* alloc */ + pr_alloc (2, bt); + + /* fill the arrays up and down */ + memcpy(bt->param[bt->nr].c, b->c, sizeof(b->c)); + memcpy(bt->param[bt->nr].a, b->a, sizeof(b->a)); + memcpy(bt->param[bt->nr+1].c, b->c, sizeof(b->c)); + + /* The definitions of linear angles depend on the order of atoms, + * that means that for atoms i-j-k, with certain parameter a, the + * corresponding k-j-i angle will have parameter 1-a. + */ + if (ftype == F_LINEAR_ANGLES) + { + bt->param[bt->nr+1].c[0] = 1-bt->param[bt->nr+1].c[0]; + bt->param[bt->nr+1].c[2] = 1-bt->param[bt->nr+1].c[2]; + } + + for (j = 0; (j < nral); j++) + { + bt->param[bt->nr+1].a[j] = b->a[nral-1-j]; + } + + bt->nr += 2; + } +} + +void push_bt(directive d, t_params bt[], int nral, + gpp_atomtype_t at, + t_bond_atomtype bat, char *line, + warninp_t wi) +{ + const char *formal[MAXATOMLIST+1] = { + "%s", + "%s%s", + "%s%s%s", + "%s%s%s%s", + "%s%s%s%s%s", + "%s%s%s%s%s%s", + "%s%s%s%s%s%s%s" + }; + const char *formnl[MAXATOMLIST+1] = { + "%*s", + "%*s%*s", + "%*s%*s%*s", + "%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s%*s" + }; + const char *formlf = "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf"; + int i, ft, ftype, nn, nrfp, nrfpA; + char f1[STRLEN]; + char alc[MAXATOMLIST+1][20]; + /* One force parameter more, so we can check if we read too many */ + double c[MAXFORCEPARAM+1]; + t_param p; + char errbuf[256]; + + if ((bat && at) || (!bat && !at)) + { + gmx_incons("You should pass either bat or at to push_bt"); + } + + /* Make format string (nral ints+functype) */ + if ((nn = sscanf(line, formal[nral], + alc[0], alc[1], alc[2], alc[3], alc[4], alc[5])) != nral+1) + { + sprintf(errbuf, "Not enough atomtypes (%d instead of %d)", nn-1, nral); + warning_error(wi, errbuf); + return; + } + + ft = strtol(alc[nral], NULL, 10); + ftype = ifunc_index(d, ft); + nrfp = NRFP(ftype); + nrfpA = interaction_function[ftype].nrfpA; + strcpy(f1, formnl[nral]); + strcat(f1, formlf); + if ((nn = sscanf(line, f1, &c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], &c[7], &c[8], &c[9], &c[10], &c[11], &c[12])) + != nrfp) + { + if (nn == nrfpA) + { + /* Copy the B-state from the A-state */ + copy_B_from_A(ftype, c); + } + else + { + if (nn < nrfpA) + { + warning_error(wi, "Not enough parameters"); + } + else if (nn > nrfpA && nn < nrfp) + { + warning_error(wi, "Too many parameters or not enough parameters for topology B"); + } + else if (nn > nrfp) + { + warning_error(wi, "Too many parameters"); + } + for (i = nn; (i < nrfp); i++) + { + c[i] = 0.0; + } + } + } + for (i = 0; (i < nral); i++) + { + if (at && ((p.a[i] = get_atomtype_type(alc[i], at)) == NOTSET)) + { + gmx_fatal(FARGS, "Unknown atomtype %s\n", alc[i]); + } + else if (bat && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET)) + { + gmx_fatal(FARGS, "Unknown bond_atomtype %s\n", alc[i]); + } + } + for (i = 0; (i < nrfp); i++) + { + p.c[i] = c[i]; + } + push_bondtype (&(bt[ftype]), &p, nral, ftype, FALSE, line, wi); +} + + +void push_dihedraltype(directive d, t_params bt[], + t_bond_atomtype bat, char *line, + warninp_t wi) +{ + const char *formal[MAXATOMLIST+1] = { + "%s", + "%s%s", + "%s%s%s", + "%s%s%s%s", + "%s%s%s%s%s", + "%s%s%s%s%s%s", + "%s%s%s%s%s%s%s" + }; + const char *formnl[MAXATOMLIST+1] = { + "%*s", + "%*s%*s", + "%*s%*s%*s", + "%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s%*s" + }; + const char *formlf[MAXFORCEPARAM] = { + "%lf", + "%lf%lf", + "%lf%lf%lf", + "%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf", + "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf", + }; + int i, ft, ftype, nn, nrfp, nrfpA, nral; + char f1[STRLEN]; + char alc[MAXATOMLIST+1][20]; + double c[MAXFORCEPARAM]; + t_param p; + gmx_bool bAllowRepeat; + char errbuf[256]; + + /* This routine accepts dihedraltypes defined from either 2 or 4 atoms. + * + * We first check for 2 atoms with the 3th column being an integer + * defining the type. If this isn't the case, we try it with 4 atoms + * and the 5th column defining the dihedral type. + */ + nn = sscanf(line, formal[4], alc[0], alc[1], alc[2], alc[3], alc[4]); + if (nn >= 3 && strlen(alc[2]) == 1 && isdigit(alc[2][0])) + { + nral = 2; + ft = strtol(alc[nral], NULL, 10); + /* Move atom types around a bit and use 'X' for wildcard atoms + * to create a 4-atom dihedral definition with arbitrary atoms in + * position 1 and 4. + */ + if (alc[2][0] == '2') + { + /* improper - the two atomtypes are 1,4. Use wildcards for 2,3 */ + strcpy(alc[3], alc[1]); + sprintf(alc[2], "X"); + sprintf(alc[1], "X"); + /* alc[0] stays put */ + } + else + { + /* proper - the two atomtypes are 2,3. Use wildcards for 1,4 */ + sprintf(alc[3], "X"); + strcpy(alc[2], alc[1]); + strcpy(alc[1], alc[0]); + sprintf(alc[0], "X"); + } + } + else if (nn == 5 && strlen(alc[4]) == 1 && isdigit(alc[4][0])) + { + nral = 4; + ft = strtol(alc[nral], NULL, 10); + } + else + { + sprintf(errbuf, "Incorrect number of atomtypes for dihedral (%d instead of 2 or 4)", nn-1); + warning_error(wi, errbuf); + return; + } + + if (ft == 9) + { + /* Previously, we have always overwritten parameters if e.g. a torsion + with the same atomtypes occurs on multiple lines. However, CHARMM and + some other force fields specify multiple dihedrals over some bonds, + including cosines with multiplicity 6 and somethimes even higher. + Thus, they cannot be represented with Ryckaert-Bellemans terms. + To add support for these force fields, Dihedral type 9 is identical to + normal proper dihedrals, but repeated entries are allowed. + */ + bAllowRepeat = TRUE; + ft = 1; + } + else + { + bAllowRepeat = FALSE; + } + + + ftype = ifunc_index(d, ft); + nrfp = NRFP(ftype); + nrfpA = interaction_function[ftype].nrfpA; + + strcpy(f1, formnl[nral]); + strcat(f1, formlf[nrfp-1]); + + /* Check number of parameters given */ + if ((nn = sscanf(line, f1, &c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], &c[7], &c[8], &c[9], &c[10], &c[11])) + != nrfp) + { + if (nn == nrfpA) + { + /* Copy the B-state from the A-state */ + copy_B_from_A(ftype, c); + } + else + { + if (nn < nrfpA) + { + warning_error(wi, "Not enough parameters"); + } + else if (nn > nrfpA && nn < nrfp) + { + warning_error(wi, "Too many parameters or not enough parameters for topology B"); + } + else if (nn > nrfp) + { + warning_error(wi, "Too many parameters"); + } + for (i = nn; (i < nrfp); i++) + { + c[i] = 0.0; + } + } + } + + for (i = 0; (i < 4); i++) + { + if (!strcmp(alc[i], "X")) + { + p.a[i] = -1; + } + else + { + if ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET) + { + gmx_fatal(FARGS, "Unknown bond_atomtype %s", alc[i]); + } + } + } + for (i = 0; (i < nrfp); i++) + { + p.c[i] = c[i]; + } + /* Always use 4 atoms here, since we created two wildcard atoms + * if there wasn't of them 4 already. + */ + push_bondtype (&(bt[ftype]), &p, 4, ftype, bAllowRepeat, line, wi); +} + + +void push_nbt(directive d, t_nbparam **nbt, gpp_atomtype_t atype, + char *pline, int nb_funct, + warninp_t wi) +{ + /* swap the atoms */ + const char *form3 = "%*s%*s%*s%lf%lf%lf"; + const char *form4 = "%*s%*s%*s%lf%lf%lf%lf"; + const char *form5 = "%*s%*s%*s%lf%lf%lf%lf%lf"; + char a0[80], a1[80]; + int i, f, n, ftype, nrfp; + double c[4], dum; + real cr[4]; + int ai, aj; + t_nbparam *nbp; + gmx_bool bId; + char errbuf[256]; + + if (sscanf (pline, "%s%s%d", a0, a1, &f) != 3) + { + too_few(wi); + return; + } + + ftype = ifunc_index(d, f); + + if (ftype != nb_funct) + { + sprintf(errbuf, "Trying to add %s while the default nonbond type is %s", + interaction_function[ftype].longname, + interaction_function[nb_funct].longname); + warning_error(wi, errbuf); + return; + } + + /* Get the force parameters */ + nrfp = NRFP(ftype); + if (ftype == F_LJ14) + { + n = sscanf(pline, form4, &c[0], &c[1], &c[2], &c[3]); + if (n < 2) + { + too_few(wi); + return; + } + /* When the B topology parameters are not set, + * copy them from topology A + */ + GMX_ASSERT(nrfp <= 4, "LJ-14 cannot have more than 4 parameters"); + for (i = n; i < nrfp; i++) + { + c[i] = c[i-2]; + } + } + else if (ftype == F_LJC14_Q) + { + n = sscanf(pline, form5, &c[0], &c[1], &c[2], &c[3], &dum); + if (n != 4) + { + incorrect_n_param(wi); + return; + } + } + else if (nrfp == 2) + { + if (sscanf(pline, form3, &c[0], &c[1], &dum) != 2) + { + incorrect_n_param(wi); + return; + } + } + else if (nrfp == 3) + { + if (sscanf(pline, form4, &c[0], &c[1], &c[2], &dum) != 3) + { + incorrect_n_param(wi); + return; + } + } + else + { + gmx_fatal(FARGS, "Number of force parameters for nonbonded interactions is %d" + " in file %s, line %d", nrfp, __FILE__, __LINE__); + } + for (i = 0; (i < nrfp); i++) + { + cr[i] = c[i]; + } + + /* Put the parameters in the matrix */ + if ((ai = get_atomtype_type (a0, atype)) == NOTSET) + { + gmx_fatal(FARGS, "Atomtype %s not found", a0); + } + if ((aj = get_atomtype_type (a1, atype)) == NOTSET) + { + gmx_fatal(FARGS, "Atomtype %s not found", a1); + } + nbp = &(nbt[std::max(ai, aj)][std::min(ai, aj)]); + + if (nbp->bSet) + { + bId = TRUE; + for (i = 0; i < nrfp; i++) + { + bId = bId && (nbp->c[i] == cr[i]); + } + if (!bId) + { + sprintf(errbuf, "Overriding non-bonded parameters,"); + warning(wi, errbuf); + fprintf(stderr, " old:"); + for (i = 0; i < nrfp; i++) + { + fprintf(stderr, " %g", nbp->c[i]); + } + fprintf(stderr, " new\n%s\n", pline); + } + } + nbp->bSet = TRUE; + for (i = 0; i < nrfp; i++) + { + nbp->c[i] = cr[i]; + } +} + +void +push_gb_params (gpp_atomtype_t at, char *line, + warninp_t wi) +{ + int atype; + double radius, vol, surftens, gb_radius, S_hct; + char atypename[STRLEN]; + char errbuf[STRLEN]; + + if ( (sscanf(line, "%s%lf%lf%lf%lf%lf", atypename, &radius, &vol, &surftens, &gb_radius, &S_hct)) != 6) + { + sprintf(errbuf, "Too few gb parameters for type %s\n", atypename); + warning(wi, errbuf); + } + + /* Search for atomtype */ + atype = get_atomtype_type(atypename, at); + + if (atype == NOTSET) + { + printf("Couldn't find topology match for atomtype %s\n", atypename); + abort(); + } + + set_atomtype_gbparam(at, atype, radius, vol, surftens, gb_radius, S_hct); +} + +void +push_cmaptype(directive d, t_params bt[], int nral, gpp_atomtype_t at, + t_bond_atomtype bat, char *line, + warninp_t wi) +{ + const char *formal = "%s%s%s%s%s%s%s%s"; + + int i, ft, ftype, nn, nrfp, nrfpA, nrfpB; + int start; + int nxcmap, nycmap, ncmap, read_cmap, sl, nct; + char s[20], alc[MAXATOMLIST+2][20]; + t_param p; + char errbuf[256]; + + /* Keep the compiler happy */ + read_cmap = 0; + start = 0; + + if ((nn = sscanf(line, formal, alc[0], alc[1], alc[2], alc[3], alc[4], alc[5], alc[6], alc[7])) != nral+3) + { + sprintf(errbuf, "Incorrect number of atomtypes for cmap (%d instead of 5)", nn-1); + warning_error(wi, errbuf); + return; + } + + /* Compute an offset for each line where the cmap parameters start + * ie. where the atom types and grid spacing information ends + */ + for (i = 0; i < nn; i++) + { + start += (int)strlen(alc[i]); + } + + /* There are nn-1 spaces between the atom types and the grid spacing info in the cmap.itp file */ + /* start is the position on the line where we start to read the actual cmap grid data from the itp file */ + start = start + nn -1; + + ft = strtol(alc[nral], NULL, 10); + nxcmap = strtol(alc[nral+1], NULL, 10); + nycmap = strtol(alc[nral+2], NULL, 10); + + /* Check for equal grid spacing in x and y dims */ + if (nxcmap != nycmap) + { + gmx_fatal(FARGS, "Not the same grid spacing in x and y for cmap grid: x=%d, y=%d", nxcmap, nycmap); + } + + ncmap = nxcmap*nycmap; + ftype = ifunc_index(d, ft); + nrfpA = strtol(alc[6], NULL, 10)*strtol(alc[6], NULL, 10); + nrfpB = strtol(alc[7], NULL, 10)*strtol(alc[7], NULL, 10); + nrfp = nrfpA+nrfpB; + + /* Allocate memory for the CMAP grid */ + bt[F_CMAP].ncmap += nrfp; + srenew(bt[F_CMAP].cmap, bt[F_CMAP].ncmap); + + /* Read in CMAP parameters */ + sl = 0; + for (i = 0; i < ncmap; i++) + { + while (isspace(*(line+start+sl))) + { + sl++; + } + nn = sscanf(line+start+sl, " %s ", s); + sl += strlen(s); + bt[F_CMAP].cmap[i+(bt[F_CMAP].ncmap)-nrfp] = strtod(s, NULL); + + if (nn == 1) + { + read_cmap++; + } + else + { + gmx_fatal(FARGS, "Error in reading cmap parameter for angle %s %s %s %s %s", alc[0], alc[1], alc[2], alc[3], alc[4]); + } + + } + + /* Check do that we got the number of parameters we expected */ + if (read_cmap == nrfpA) + { + for (i = 0; i < ncmap; i++) + { + bt[F_CMAP].cmap[i+ncmap] = bt[F_CMAP].cmap[i]; + } + } + else + { + if (read_cmap < nrfpA) + { + warning_error(wi, "Not enough cmap parameters"); + } + else if (read_cmap > nrfpA && read_cmap < nrfp) + { + warning_error(wi, "Too many cmap parameters or not enough parameters for topology B"); + } + else if (read_cmap > nrfp) + { + warning_error(wi, "Too many cmap parameters"); + } + } + + + /* Set grid spacing and the number of grids (we assume these numbers to be the same for all grids + * so we can safely assign them each time + */ + bt[F_CMAP].grid_spacing = nxcmap; /* Or nycmap, they need to be equal */ + bt[F_CMAP].nc = bt[F_CMAP].nc + 1; /* Since we are incrementing here, we need to subtract later, see (*****) */ + nct = (nral+1) * bt[F_CMAP].nc; + + /* Allocate memory for the cmap_types information */ + srenew(bt[F_CMAP].cmap_types, nct); + + for (i = 0; (i < nral); i++) + { + if (at && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET)) + { + gmx_fatal(FARGS, "Unknown atomtype %s\n", alc[i]); + } + else if (bat && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET)) + { + gmx_fatal(FARGS, "Unknown bond_atomtype %s\n", alc[i]); + } + + /* Assign a grid number to each cmap_type */ + bt[F_CMAP].cmap_types[bt[F_CMAP].nct++] = get_bond_atomtype_type(alc[i], bat); + } + + /* Assign a type number to this cmap */ + bt[F_CMAP].cmap_types[bt[F_CMAP].nct++] = bt[F_CMAP].nc-1; /* Since we inremented earlier, we need to subtrac here, to get the types right (****) */ + + /* Check for the correct number of atoms (again) */ + if (bt[F_CMAP].nct != nct) + { + gmx_fatal(FARGS, "Incorrect number of atom types (%d) in cmap type %d\n", nct, bt[F_CMAP].nc); + } + + /* Is this correct?? */ + for (i = 0; i < MAXFORCEPARAM; i++) + { + p.c[i] = NOTSET; + } + + /* Push the bond to the bondlist */ + push_bondtype (&(bt[ftype]), &p, nral, ftype, FALSE, line, wi); +} + + +static void push_atom_now(t_symtab *symtab, t_atoms *at, int atomnr, + int atomicnumber, + int type, char *ctype, int ptype, + char *resnumberic, + char *resname, char *name, real m0, real q0, + int typeB, char *ctypeB, real mB, real qB) +{ + int j, resind = 0, resnr; + unsigned char ric; + int nr = at->nr; + + if (((nr == 0) && (atomnr != 1)) || (nr && (atomnr != at->nr+1))) + { + gmx_fatal(FARGS, "Atoms in the .top are not numbered consecutively from 1 (rather, atomnr = %d, while at->nr = %d)", atomnr, at->nr); + } + + j = strlen(resnumberic) - 1; + if (isdigit(resnumberic[j])) + { + ric = ' '; + } + else + { + ric = resnumberic[j]; + if (j == 0 || !isdigit(resnumberic[j-1])) + { + gmx_fatal(FARGS, "Invalid residue number '%s' for atom %d", + resnumberic, atomnr); + } + } + resnr = strtol(resnumberic, NULL, 10); + + if (nr > 0) + { + resind = at->atom[nr-1].resind; + } + if (nr == 0 || strcmp(resname, *at->resinfo[resind].name) != 0 || + resnr != at->resinfo[resind].nr || + ric != at->resinfo[resind].ic) + { + if (nr == 0) + { + resind = 0; + } + else + { + resind++; + } + at->nres = resind + 1; + srenew(at->resinfo, at->nres); + at->resinfo[resind].name = put_symtab(symtab, resname); + at->resinfo[resind].nr = resnr; + at->resinfo[resind].ic = ric; + } + else + { + resind = at->atom[at->nr-1].resind; + } + + /* New atom instance + * get new space for arrays + */ + srenew(at->atom, nr+1); + srenew(at->atomname, nr+1); + srenew(at->atomtype, nr+1); + srenew(at->atomtypeB, nr+1); + + /* fill the list */ + at->atom[nr].type = type; + at->atom[nr].ptype = ptype; + at->atom[nr].q = q0; + at->atom[nr].m = m0; + at->atom[nr].typeB = typeB; + at->atom[nr].qB = qB; + at->atom[nr].mB = mB; + + at->atom[nr].resind = resind; + at->atom[nr].atomnumber = atomicnumber; + at->atomname[nr] = put_symtab(symtab, name); + at->atomtype[nr] = put_symtab(symtab, ctype); + at->atomtypeB[nr] = put_symtab(symtab, ctypeB); + at->nr++; +} + +void push_cg(t_block *block, int *lastindex, int index, int a) +{ + if (debug) + { + fprintf (debug, "Index %d, Atom %d\n", index, a); + } + + if (((block->nr) && (*lastindex != index)) || (!block->nr)) + { + /* add a new block */ + block->nr++; + srenew(block->index, block->nr+1); + } + block->index[block->nr] = a + 1; + *lastindex = index; +} + +void push_atom(t_symtab *symtab, t_block *cgs, + t_atoms *at, gpp_atomtype_t atype, char *line, int *lastcg, + warninp_t wi) +{ + int nr, ptype; + int cgnumber, atomnr, type, typeB, nscan; + char id[STRLEN], ctype[STRLEN], ctypeB[STRLEN], + resnumberic[STRLEN], resname[STRLEN], name[STRLEN], check[STRLEN]; + double m, q, mb, qb; + real m0, q0, mB, qB; + + /* Make a shortcut for writing in this molecule */ + nr = at->nr; + + /* Fixed parameters */ + if (sscanf(line, "%s%s%s%s%s%d", + id, ctype, resnumberic, resname, name, &cgnumber) != 6) + { + too_few(wi); + return; + } + sscanf(id, "%d", &atomnr); + if ((type = get_atomtype_type(ctype, atype)) == NOTSET) + { + gmx_fatal(FARGS, "Atomtype %s not found", ctype); + } + ptype = get_atomtype_ptype(type, atype); + + /* Set default from type */ + q0 = get_atomtype_qA(type, atype); + m0 = get_atomtype_massA(type, atype); + typeB = type; + qB = q0; + mB = m0; + + /* Optional parameters */ + nscan = sscanf(line, "%*s%*s%*s%*s%*s%*s%lf%lf%s%lf%lf%s", + &q, &m, ctypeB, &qb, &mb, check); + + /* Nasty switch that falls thru all the way down! */ + if (nscan > 0) + { + q0 = qB = q; + if (nscan > 1) + { + m0 = mB = m; + if (nscan > 2) + { + if ((typeB = get_atomtype_type(ctypeB, atype)) == NOTSET) + { + gmx_fatal(FARGS, "Atomtype %s not found", ctypeB); + } + qB = get_atomtype_qA(typeB, atype); + mB = get_atomtype_massA(typeB, atype); + if (nscan > 3) + { + qB = qb; + if (nscan > 4) + { + mB = mb; + if (nscan > 5) + { + warning_error(wi, "Too many parameters"); + } + } + } + } + } + } + if (debug) + { + fprintf(debug, "mB=%g, qB=%g, typeB=%d\n", mB, qB, typeB); + } + + push_cg(cgs, lastcg, cgnumber, nr); + + push_atom_now(symtab, at, atomnr, get_atomtype_atomnumber(type, atype), + type, ctype, ptype, resnumberic, + resname, name, m0, q0, typeB, + typeB == type ? ctype : ctypeB, mB, qB); +} + +void push_molt(t_symtab *symtab, int *nmol, t_molinfo **mol, char *line, + warninp_t wi) +{ + char type[STRLEN]; + int nrexcl, i; + t_molinfo *newmol; + + if ((sscanf(line, "%s%d", type, &nrexcl)) != 2) + { + warning_error(wi, "Expected a molecule type name and nrexcl"); + } + + /* Test if this moleculetype overwrites another */ + i = 0; + while (i < *nmol) + { + if (strcmp(*((*mol)[i].name), type) == 0) + { + gmx_fatal(FARGS, "moleculetype %s is redefined", type); + } + i++; + } + + (*nmol)++; + srenew(*mol, *nmol); + newmol = &((*mol)[*nmol-1]); + init_molinfo(newmol); + + /* Fill in the values */ + newmol->name = put_symtab(symtab, type); + newmol->nrexcl = nrexcl; + newmol->excl_set = FALSE; +} + +static gmx_bool default_nb_params(int ftype, t_params bt[], t_atoms *at, + t_param *p, int c_start, gmx_bool bB, gmx_bool bGenPairs) +{ + int i, j, ti, tj, ntype; + gmx_bool bFound; + t_param *pi = NULL; + int nr = bt[ftype].nr; + int nral = NRAL(ftype); + int nrfp = interaction_function[ftype].nrfpA; + int nrfpB = interaction_function[ftype].nrfpB; + + if ((!bB && nrfp == 0) || (bB && nrfpB == 0)) + { + return TRUE; + } + + bFound = FALSE; + if (bGenPairs) + { + /* First test the generated-pair position to save + * time when we have 1000*1000 entries for e.g. OPLS... + */ + ntype = static_cast(std::sqrt(static_cast(nr))); + GMX_ASSERT(ntype * ntype == nr, "Number of pairs of generated non-bonded parameters should be a perfect square"); + if (bB) + { + ti = at->atom[p->a[0]].typeB; + tj = at->atom[p->a[1]].typeB; + } + else + { + ti = at->atom[p->a[0]].type; + tj = at->atom[p->a[1]].type; + } + pi = &(bt[ftype].param[ntype*ti+tj]); + bFound = ((ti == pi->a[0]) && (tj == pi->a[1])); + } + + /* Search explicitly if we didnt find it */ + if (!bFound) + { + for (i = 0; ((i < nr) && !bFound); i++) + { + pi = &(bt[ftype].param[i]); + if (bB) + { + for (j = 0; ((j < nral) && + (at->atom[p->a[j]].typeB == pi->a[j])); j++) + { + ; + } + } + else + { + for (j = 0; ((j < nral) && + (at->atom[p->a[j]].type == pi->a[j])); j++) + { + ; + } + } + bFound = (j == nral); + } + } + + if (bFound) + { + if (bB) + { + if (nrfp+nrfpB > MAXFORCEPARAM) + { + gmx_incons("Too many force parameters"); + } + for (j = c_start; (j < nrfpB); j++) + { + p->c[nrfp+j] = pi->c[j]; + } + } + else + { + for (j = c_start; (j < nrfp); j++) + { + p->c[j] = pi->c[j]; + } + } + } + else + { + for (j = c_start; (j < nrfp); j++) + { + p->c[j] = 0.0; + } + } + return bFound; +} + +static gmx_bool default_cmap_params(t_params bondtype[], + t_atoms *at, gpp_atomtype_t atype, + t_param *p, gmx_bool bB, + int *cmap_type, int *nparam_def) +{ + int i, nparam_found; + int ct; + gmx_bool bFound = FALSE; + + nparam_found = 0; + ct = 0; + + /* Match the current cmap angle against the list of cmap_types */ + for (i = 0; i < bondtype[F_CMAP].nct && !bFound; i += 6) + { + if (bB) + { + + } + else + { + if ( + (get_atomtype_batype(at->atom[p->a[0]].type, atype) == bondtype[F_CMAP].cmap_types[i]) && + (get_atomtype_batype(at->atom[p->a[1]].type, atype) == bondtype[F_CMAP].cmap_types[i+1]) && + (get_atomtype_batype(at->atom[p->a[2]].type, atype) == bondtype[F_CMAP].cmap_types[i+2]) && + (get_atomtype_batype(at->atom[p->a[3]].type, atype) == bondtype[F_CMAP].cmap_types[i+3]) && + (get_atomtype_batype(at->atom[p->a[4]].type, atype) == bondtype[F_CMAP].cmap_types[i+4])) + { + /* Found cmap torsion */ + bFound = TRUE; + ct = bondtype[F_CMAP].cmap_types[i+5]; + nparam_found = 1; + } + } + } + + /* If we did not find a matching type for this cmap torsion */ + if (!bFound) + { + gmx_fatal(FARGS, "Unknown cmap torsion between atoms %d %d %d %d %d\n", + p->a[0]+1, p->a[1]+1, p->a[2]+1, p->a[3]+1, p->a[4]+1); + } + + *nparam_def = nparam_found; + *cmap_type = ct; + + return bFound; +} + +static gmx_bool default_params(int ftype, t_params bt[], + t_atoms *at, gpp_atomtype_t atype, + t_param *p, gmx_bool bB, + t_param **param_def, + int *nparam_def) +{ + int i, j, nparam_found; + gmx_bool bFound, bSame; + t_param *pi = NULL; + t_param *pj = NULL; + int nr = bt[ftype].nr; + int nral = NRAL(ftype); + int nrfpA = interaction_function[ftype].nrfpA; + int nrfpB = interaction_function[ftype].nrfpB; + + if ((!bB && nrfpA == 0) || (bB && nrfpB == 0)) + { + return TRUE; + } + + + /* We allow wildcards now. The first type (with or without wildcards) that + * fits is used, so you should probably put the wildcarded bondtypes + * at the end of each section. + */ + bFound = FALSE; + nparam_found = 0; + /* OPLS uses 1000s of dihedraltypes, so in order to speed up the scanning we have a + * special case for this. Check for B state outside loop to speed it up. + */ + if (ftype == F_PDIHS || ftype == F_RBDIHS || ftype == F_IDIHS || ftype == F_PIDIHS) + { + if (bB) + { + for (i = 0; ((i < nr) && !bFound); i++) + { + pi = &(bt[ftype].param[i]); + bFound = + ( + ((pi->ai() == -1) || (get_atomtype_batype(at->atom[p->ai()].typeB, atype) == pi->ai())) && + ((pi->aj() == -1) || (get_atomtype_batype(at->atom[p->aj()].typeB, atype) == pi->aj())) && + ((pi->ak() == -1) || (get_atomtype_batype(at->atom[p->ak()].typeB, atype) == pi->ak())) && + ((pi->al() == -1) || (get_atomtype_batype(at->atom[p->al()].typeB, atype) == pi->al())) + ); + } + } + else + { + /* State A */ + for (i = 0; ((i < nr) && !bFound); i++) + { + pi = &(bt[ftype].param[i]); + bFound = + ( + ((pi->ai() == -1) || (get_atomtype_batype(at->atom[p->ai()].type, atype) == pi->ai())) && + ((pi->aj() == -1) || (get_atomtype_batype(at->atom[p->aj()].type, atype) == pi->aj())) && + ((pi->ak() == -1) || (get_atomtype_batype(at->atom[p->ak()].type, atype) == pi->ak())) && + ((pi->al() == -1) || (get_atomtype_batype(at->atom[p->al()].type, atype) == pi->al())) + ); + } + } + /* Find additional matches for this dihedral - necessary for ftype==9 which is used e.g. for charmm. + * The rules in that case is that additional matches HAVE to be on adjacent lines! + */ + if (bFound == TRUE) + { + nparam_found++; + bSame = TRUE; + /* Continue from current i value */ + for (j = i+1; j < nr && bSame; j += 2) + { + pj = &(bt[ftype].param[j]); + bSame = (pi->ai() == pj->ai() && pi->aj() == pj->aj() && pi->ak() == pj->ak() && pi->al() == pj->al()); + if (bSame) + { + nparam_found++; + } + /* nparam_found will be increased as long as the numbers match */ + } + } + } + else /* Not a dihedral */ + { + for (i = 0; ((i < nr) && !bFound); i++) + { + pi = &(bt[ftype].param[i]); + if (bB) + { + for (j = 0; ((j < nral) && + (get_atomtype_batype(at->atom[p->a[j]].typeB, atype) == pi->a[j])); j++) + { + ; + } + } + else + { + for (j = 0; ((j < nral) && + (get_atomtype_batype(at->atom[p->a[j]].type, atype) == pi->a[j])); j++) + { + ; + } + } + bFound = (j == nral); + } + if (bFound) + { + nparam_found = 1; + } + } + + *param_def = pi; + *nparam_def = nparam_found; + + return bFound; +} + + + +void push_bond(directive d, t_params bondtype[], t_params bond[], + t_atoms *at, gpp_atomtype_t atype, char *line, + gmx_bool bBonded, gmx_bool bGenPairs, real fudgeQQ, + gmx_bool bZero, gmx_bool *bWarn_copy_A_B, + warninp_t wi) +{ + const char *aaformat[MAXATOMLIST] = { + "%d%d", + "%d%d%d", + "%d%d%d%d", + "%d%d%d%d%d", + "%d%d%d%d%d%d", + "%d%d%d%d%d%d%d" + }; + const char *asformat[MAXATOMLIST] = { + "%*s%*s", + "%*s%*s%*s", + "%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s", + "%*s%*s%*s%*s%*s%*s%*s" + }; + const char *ccformat = "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf"; + int nr, i, j, nral, nral_fmt, nread, ftype; + char format[STRLEN]; + /* One force parameter more, so we can check if we read too many */ + double cc[MAXFORCEPARAM+1]; + int aa[MAXATOMLIST+1]; + t_param param, *param_defA, *param_defB; + gmx_bool bFoundA = FALSE, bFoundB = FALSE, bDef, bPert, bSwapParity = FALSE; + int nparam_defA, nparam_defB; + char errbuf[256]; + + nparam_defA = nparam_defB = 0; + + ftype = ifunc_index(d, 1); + nral = NRAL(ftype); + for (j = 0; j < MAXATOMLIST; j++) + { + aa[j] = NOTSET; + } + bDef = (NRFP(ftype) > 0); + + if (ftype == F_SETTLE) + { + /* SETTLE acts on 3 atoms, but the topology format only specifies + * the first atom (for historical reasons). + */ + nral_fmt = 1; + } + else + { + nral_fmt = nral; + } + + nread = sscanf(line, aaformat[nral_fmt-1], + &aa[0], &aa[1], &aa[2], &aa[3], &aa[4], &aa[5]); + + if (ftype == F_SETTLE) + { + aa[3] = aa[1]; + aa[1] = aa[0] + 1; + aa[2] = aa[0] + 2; + } + + if (nread < nral_fmt) + { + too_few(wi); + return; + } + else if (nread > nral_fmt) + { + /* this is a hack to allow for virtual sites with swapped parity */ + bSwapParity = (aa[nral] < 0); + if (bSwapParity) + { + aa[nral] = -aa[nral]; + } + ftype = ifunc_index(d, aa[nral]); + if (bSwapParity) + { + switch (ftype) + { + case F_VSITE3FAD: + case F_VSITE3OUT: + break; + default: + gmx_fatal(FARGS, "Negative function types only allowed for %s and %s", + interaction_function[F_VSITE3FAD].longname, + interaction_function[F_VSITE3OUT].longname); + } + } + } + + + /* Check for double atoms and atoms out of bounds */ + for (i = 0; (i < nral); i++) + { + if (aa[i] < 1 || aa[i] > at->nr) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + "Atom index (%d) in %s out of bounds (1-%d).\n" + "This probably means that you have inserted topology section \"%s\"\n" + "in a part belonging to a different molecule than you intended to.\n" + "In that case move the \"%s\" section to the right molecule.", + get_warning_file(wi), get_warning_line(wi), + aa[i], dir2str(d), at->nr, dir2str(d), dir2str(d)); + } + for (j = i+1; (j < nral); j++) + { + if (aa[i] == aa[j]) + { + sprintf(errbuf, "Duplicate atom index (%d) in %s", aa[i], dir2str(d)); + warning(wi, errbuf); + } + } + } + + /* default force parameters */ + for (j = 0; (j < MAXATOMLIST); j++) + { + param.a[j] = aa[j]-1; + } + for (j = 0; (j < MAXFORCEPARAM); j++) + { + param.c[j] = 0.0; + } + + /* Get force params for normal and free energy perturbation + * studies, as determined by types! + */ + + if (bBonded) + { + bFoundA = default_params(ftype, bondtype, at, atype, ¶m, FALSE, ¶m_defA, &nparam_defA); + if (bFoundA) + { + /* Copy the A-state and B-state default parameters. */ + GMX_ASSERT(NRFPA(ftype)+NRFPB(ftype) <= MAXFORCEPARAM, "Bonded interactions may have at most 12 parameters"); + for (j = 0; (j < NRFPA(ftype)+NRFPB(ftype)); j++) + { + param.c[j] = param_defA->c[j]; + } + } + bFoundB = default_params(ftype, bondtype, at, atype, ¶m, TRUE, ¶m_defB, &nparam_defB); + if (bFoundB) + { + /* Copy only the B-state default parameters */ + for (j = NRFPA(ftype); (j < NRFP(ftype)); j++) + { + param.c[j] = param_defB->c[j]; + } + } + } + else if (ftype == F_LJ14) + { + bFoundA = default_nb_params(ftype, bondtype, at, ¶m, 0, FALSE, bGenPairs); + bFoundB = default_nb_params(ftype, bondtype, at, ¶m, 0, TRUE, bGenPairs); + } + else if (ftype == F_LJC14_Q) + { + param.c[0] = fudgeQQ; + /* Fill in the A-state charges as default parameters */ + param.c[1] = at->atom[param.a[0]].q; + param.c[2] = at->atom[param.a[1]].q; + /* The default LJ parameters are the standard 1-4 parameters */ + bFoundA = default_nb_params(F_LJ14, bondtype, at, ¶m, 3, FALSE, bGenPairs); + bFoundB = TRUE; + } + else if (ftype == F_LJC_PAIRS_NB) + { + /* Defaults are not supported here */ + bFoundA = FALSE; + bFoundB = TRUE; + } + else + { + gmx_incons("Unknown function type in push_bond"); + } + + if (nread > nral_fmt) + { + /* Manually specified parameters - in this case we discard multiple torsion info! */ + + strcpy(format, asformat[nral_fmt-1]); + strcat(format, ccformat); + + nread = sscanf(line, format, &cc[0], &cc[1], &cc[2], &cc[3], &cc[4], &cc[5], + &cc[6], &cc[7], &cc[8], &cc[9], &cc[10], &cc[11], &cc[12]); + + if ((nread == NRFPA(ftype)) && (NRFPB(ftype) != 0)) + { + /* We only have to issue a warning if these atoms are perturbed! */ + bPert = FALSE; + for (j = 0; (j < nral); j++) + { + bPert = bPert || PERTURBED(at->atom[param.a[j]]); + } + + if (bPert && *bWarn_copy_A_B) + { + sprintf(errbuf, + "Some parameters for bonded interaction involving perturbed atoms are specified explicitly in state A, but not B - copying A to B"); + warning(wi, errbuf); + *bWarn_copy_A_B = FALSE; + } + + /* If only the A parameters were specified, copy them to the B state */ + /* The B-state parameters correspond to the first nrfpB + * A-state parameters. + */ + for (j = 0; (j < NRFPB(ftype)); j++) + { + cc[nread++] = cc[j]; + } + } + + /* If nread was 0 or EOF, no parameters were read => use defaults. + * If nread was nrfpA we copied above so nread=nrfp. + * If nread was nrfp we are cool. + * For F_LJC14_Q we allow supplying fudgeQQ only. + * Anything else is an error! + */ + if ((nread != 0) && (nread != EOF) && (nread != NRFP(ftype)) && + !(ftype == F_LJC14_Q && nread == 1)) + { + gmx_fatal(FARGS, "Incorrect number of parameters - found %d, expected %d or %d for %s.", + nread, NRFPA(ftype), NRFP(ftype), + interaction_function[ftype].longname); + } + + for (j = 0; (j < nread); j++) + { + param.c[j] = cc[j]; + } + + /* Check whether we have to use the defaults */ + if (nread == NRFP(ftype)) + { + bDef = FALSE; + } + } + else + { + nread = 0; + } + /* nread now holds the number of force parameters read! */ + + if (bDef) + { + /* Use defaults */ + /* When we have multiple terms it would be very dangerous to allow perturbations to a different atom type! */ + if (ftype == F_PDIHS) + { + if ((nparam_defA != nparam_defB) || ((nparam_defA > 1 || nparam_defB > 1) && (param_defA != param_defB))) + { + sprintf(errbuf, + "Cannot automatically perturb a torsion with multiple terms to different form.\n" + "Please specify perturbed parameters manually for this torsion in your topology!"); + warning_error(wi, errbuf); + } + } + + if (nread > 0 && nread < NRFPA(ftype)) + { + /* Issue an error, do not use defaults */ + sprintf(errbuf, "Not enough parameters, there should be at least %d (or 0 for defaults)", NRFPA(ftype)); + warning_error(wi, errbuf); + } + + if (nread == 0 || nread == EOF) + { + if (!bFoundA) + { + if (interaction_function[ftype].flags & IF_VSITE) + { + /* set them to NOTSET, will be calculated later */ + for (j = 0; (j < MAXFORCEPARAM); j++) + { + param.c[j] = NOTSET; + } + + if (bSwapParity) + { + param.c1() = -1; /* flag to swap parity of vsite construction */ + } + } + else + { + if (bZero) + { + fprintf(stderr, "NOTE: No default %s types, using zeroes\n", + interaction_function[ftype].longname); + } + else + { + sprintf(errbuf, "No default %s types", interaction_function[ftype].longname); + warning_error(wi, errbuf); + } + } + } + else + { + if (bSwapParity) + { + switch (ftype) + { + case F_VSITE3FAD: + param.c0() = 360 - param.c0(); + break; + case F_VSITE3OUT: + param.c2() = -param.c2(); + break; + } + } + } + if (!bFoundB) + { + /* We only have to issue a warning if these atoms are perturbed! */ + bPert = FALSE; + for (j = 0; (j < nral); j++) + { + bPert = bPert || PERTURBED(at->atom[param.a[j]]); + } + + if (bPert) + { + sprintf(errbuf, "No default %s types for perturbed atoms, " + "using normal values", interaction_function[ftype].longname); + warning(wi, errbuf); + } + } + } + } + + if ((ftype == F_PDIHS || ftype == F_ANGRES || ftype == F_ANGRESZ) + && param.c[5] != param.c[2]) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " %s multiplicity can not be perturbed %f!=%f", + get_warning_file(wi), get_warning_line(wi), + interaction_function[ftype].longname, + param.c[2], param.c[5]); + } + + if (IS_TABULATED(ftype) && param.c[2] != param.c[0]) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " %s table number can not be perturbed %d!=%d", + get_warning_file(wi), get_warning_line(wi), + interaction_function[ftype].longname, + (int)(param.c[0]+0.5), (int)(param.c[2]+0.5)); + } + + /* Dont add R-B dihedrals where all parameters are zero (no interaction) */ + if (ftype == F_RBDIHS) + { + nr = 0; + for (i = 0; i < NRFP(ftype); i++) + { + if (param.c[i] != 0) + { + nr++; + } + } + if (nr == 0) + { + return; + } + } + + /* Put the values in the appropriate arrays */ + add_param_to_list (&bond[ftype], ¶m); + + /* Push additional torsions from FF for ftype==9 if we have them. + * We have already checked that the A/B states do not differ in this case, + * so we do not have to double-check that again, or the vsite stuff. + * In addition, those torsions cannot be automatically perturbed. + */ + if (bDef && ftype == F_PDIHS) + { + for (i = 1; i < nparam_defA; i++) + { + /* Advance pointer! */ + param_defA += 2; + for (j = 0; (j < NRFPA(ftype)+NRFPB(ftype)); j++) + { + param.c[j] = param_defA->c[j]; + } + /* And push the next term for this torsion */ + add_param_to_list (&bond[ftype], ¶m); + } + } +} + +void push_cmap(directive d, t_params bondtype[], t_params bond[], + t_atoms *at, gpp_atomtype_t atype, char *line, + warninp_t wi) +{ + const char *aaformat[MAXATOMLIST+1] = + { + "%d", + "%d%d", + "%d%d%d", + "%d%d%d%d", + "%d%d%d%d%d", + "%d%d%d%d%d%d", + "%d%d%d%d%d%d%d" + }; + + int i, j, ftype, nral, nread, ncmap_params; + int cmap_type; + int aa[MAXATOMLIST+1]; + char errbuf[256]; + gmx_bool bFound; + t_param param; + + ftype = ifunc_index(d, 1); + nral = NRAL(ftype); + ncmap_params = 0; + + nread = sscanf(line, aaformat[nral-1], + &aa[0], &aa[1], &aa[2], &aa[3], &aa[4], &aa[5]); + + if (nread < nral) + { + too_few(wi); + return; + } + else if (nread == nral) + { + ftype = ifunc_index(d, 1); + } + + /* Check for double atoms and atoms out of bounds */ + for (i = 0; i < nral; i++) + { + if (aa[i] < 1 || aa[i] > at->nr) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + "Atom index (%d) in %s out of bounds (1-%d).\n" + "This probably means that you have inserted topology section \"%s\"\n" + "in a part belonging to a different molecule than you intended to.\n" + "In that case move the \"%s\" section to the right molecule.", + get_warning_file(wi), get_warning_line(wi), + aa[i], dir2str(d), at->nr, dir2str(d), dir2str(d)); + } + + for (j = i+1; (j < nral); j++) + { + if (aa[i] == aa[j]) + { + sprintf(errbuf, "Duplicate atom index (%d) in %s", aa[i], dir2str(d)); + warning(wi, errbuf); + } + } + } + + /* default force parameters */ + for (j = 0; (j < MAXATOMLIST); j++) + { + param.a[j] = aa[j]-1; + } + for (j = 0; (j < MAXFORCEPARAM); j++) + { + param.c[j] = 0.0; + } + + /* Get the cmap type for this cmap angle */ + bFound = default_cmap_params(bondtype, at, atype, ¶m, FALSE, &cmap_type, &ncmap_params); + + /* We want exactly one parameter (the cmap type in state A (currently no state B) back */ + if (bFound && ncmap_params == 1) + { + /* Put the values in the appropriate arrays */ + param.c[0] = cmap_type; + add_param_to_list(&bond[ftype], ¶m); + } + else + { + /* This is essentially the same check as in default_cmap_params() done one more time */ + gmx_fatal(FARGS, "Unable to assign a cmap type to torsion %d %d %d %d and %d\n", + param.a[0]+1, param.a[1]+1, param.a[2]+1, param.a[3]+1, param.a[4]+1); + } +} + + + +void push_vsitesn(directive d, t_params bond[], + t_atoms *at, char *line, + warninp_t wi) +{ + char *ptr; + int type, ftype, j, n, ret, nj, a; + int *atc = NULL; + double *weight = NULL, weight_tot; + t_param param; + + /* default force parameters */ + for (j = 0; (j < MAXATOMLIST); j++) + { + param.a[j] = NOTSET; + } + for (j = 0; (j < MAXFORCEPARAM); j++) + { + param.c[j] = 0.0; + } + + ptr = line; + ret = sscanf(ptr, "%d%n", &a, &n); + ptr += n; + if (ret == 0) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " Expected an atom index in section \"%s\"", + get_warning_file(wi), get_warning_line(wi), + dir2str(d)); + } + + param.a[0] = a - 1; + + sscanf(ptr, "%d%n", &type, &n); + ptr += n; + ftype = ifunc_index(d, type); + + weight_tot = 0; + nj = 0; + do + { + ret = sscanf(ptr, "%d%n", &a, &n); + ptr += n; + if (ret > 0) + { + if (nj % 20 == 0) + { + srenew(atc, nj+20); + srenew(weight, nj+20); + } + atc[nj] = a - 1; + switch (type) + { + case 1: + weight[nj] = 1; + break; + case 2: + /* Here we use the A-state mass as a parameter. + * Note that the B-state mass has no influence. + */ + weight[nj] = at->atom[atc[nj]].m; + break; + case 3: + weight[nj] = -1; + ret = sscanf(ptr, "%lf%n", &(weight[nj]), &n); + ptr += n; + if (weight[nj] < 0) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " No weight or negative weight found for vsiten constructing atom %d (atom index %d)", + get_warning_file(wi), get_warning_line(wi), + nj+1, atc[nj]+1); + } + break; + default: + gmx_fatal(FARGS, "Unknown vsiten type %d", type); + } + weight_tot += weight[nj]; + nj++; + } + } + while (ret > 0); + + if (nj == 0) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " Expected more than one atom index in section \"%s\"", + get_warning_file(wi), get_warning_line(wi), + dir2str(d)); + } + + if (weight_tot == 0) + { + gmx_fatal(FARGS, "[ file %s, line %d ]:\n" + " The total mass of the construting atoms is zero", + get_warning_file(wi), get_warning_line(wi)); + } + + for (j = 0; j < nj; j++) + { + param.a[1] = atc[j]; + param.c[0] = nj; + param.c[1] = weight[j]/weight_tot; + /* Put the values in the appropriate arrays */ + add_param_to_list (&bond[ftype], ¶m); + } + + sfree(atc); + sfree(weight); +} + +void push_mol(int nrmols, t_molinfo mols[], char *pline, int *whichmol, + int *nrcopies, + warninp_t wi) +{ + char type[STRLEN]; + + if (sscanf(pline, "%s%d", type, nrcopies) != 2) + { + too_few(wi); + return; + } + + /* Search moleculename. + * Here we originally only did case insensitive matching. But because + * some PDB files can have many chains and use case to generate more + * chain-identifiers, which in turn end up in our moleculetype name, + * we added support for case-sensitivity. + */ + int nrcs = 0; + int nrci = 0; + int matchci = -1; + int matchcs = -1; + for (int i = 0; i < nrmols; i++) + { + if (strcmp(type, *(mols[i].name)) == 0) + { + nrcs++; + matchcs = i; + } + if (gmx_strcasecmp(type, *(mols[i].name)) == 0) + { + nrci++; + matchci = i; + } + } + + if (nrcs == 1) + { + // select the case sensitive match + *whichmol = matchcs; + } + else + { + // avoid matching case-insensitive when we have multiple matches + if (nrci > 1) + { + gmx_fatal(FARGS, "For moleculetype '%s' in [ system ] %d case insensitive matches, but %d case sensitive matches were found. Check the case of the characters in the moleculetypes.", type, nrci, nrcs); + } + if (nrci == 1) + { + // select the unique case insensitive match + *whichmol = matchci; + } + else + { + gmx_fatal(FARGS, "No such moleculetype %s", type); + } + } +} + +void init_block2(t_block2 *b2, int natom) +{ + int i; + + b2->nr = natom; + snew(b2->nra, b2->nr); + snew(b2->a, b2->nr); + for (i = 0; (i < b2->nr); i++) + { + b2->a[i] = NULL; + } +} + +void done_block2(t_block2 *b2) +{ + int i; + + if (b2->nr) + { + for (i = 0; (i < b2->nr); i++) + { + sfree(b2->a[i]); + } + sfree(b2->a); + sfree(b2->nra); + b2->nr = 0; + } +} + +void push_excl(char *line, t_block2 *b2) +{ + int i, j; + int n; + char base[STRLEN], format[STRLEN]; + + if (sscanf(line, "%d", &i) == 0) + { + return; + } + + if ((1 <= i) && (i <= b2->nr)) + { + i--; + } + else + { + if (debug) + { + fprintf(debug, "Unbound atom %d\n", i-1); + } + return; + } + strcpy(base, "%*d"); + do + { + strcpy(format, base); + strcat(format, "%d"); + n = sscanf(line, format, &j); + if (n == 1) + { + if ((1 <= j) && (j <= b2->nr)) + { + j--; + srenew(b2->a[i], ++(b2->nra[i])); + b2->a[i][b2->nra[i]-1] = j; + /* also add the reverse exclusion! */ + srenew(b2->a[j], ++(b2->nra[j])); + b2->a[j][b2->nra[j]-1] = i; + strcat(base, "%*d"); + } + else + { + gmx_fatal(FARGS, "Invalid Atomnr j: %d, b2->nr: %d\n", j, b2->nr); + } + } + } + while (n == 1); +} + +void b_to_b2(t_blocka *b, t_block2 *b2) +{ + int i; + int j, a; + + for (i = 0; (i < b->nr); i++) + { + for (j = b->index[i]; (j < b->index[i+1]); j++) + { + a = b->a[j]; + srenew(b2->a[i], ++b2->nra[i]); + b2->a[i][b2->nra[i]-1] = a; + } + } +} + +void b2_to_b(t_block2 *b2, t_blocka *b) +{ + int i, nra; + int j; + + nra = 0; + for (i = 0; (i < b2->nr); i++) + { + b->index[i] = nra; + for (j = 0; (j < b2->nra[i]); j++) + { + b->a[nra+j] = b2->a[i][j]; + } + nra += b2->nra[i]; + } + /* terminate list */ + b->index[i] = nra; +} + +static int icomp(const void *v1, const void *v2) +{ + return (*((int *) v1))-(*((int *) v2)); +} + +void merge_excl(t_blocka *excl, t_block2 *b2) +{ + int i, k; + int j; + int nra; + + if (!b2->nr) + { + return; + } + else if (b2->nr != excl->nr) + { + gmx_fatal(FARGS, "DEATH HORROR: b2->nr = %d, while excl->nr = %d", + b2->nr, excl->nr); + } + else if (debug) + { + fprintf(debug, "Entering merge_excl\n"); + } + + /* First copy all entries from excl to b2 */ + b_to_b2(excl, b2); + + /* Count and sort the exclusions */ + nra = 0; + for (i = 0; (i < b2->nr); i++) + { + if (b2->nra[i] > 0) + { + /* remove double entries */ + qsort(b2->a[i], (size_t)b2->nra[i], (size_t)sizeof(b2->a[i][0]), icomp); + k = 1; + for (j = 1; (j < b2->nra[i]); j++) + { + if (b2->a[i][j] != b2->a[i][k-1]) + { + b2->a[i][k] = b2->a[i][j]; + k++; + } + } + b2->nra[i] = k; + nra += b2->nra[i]; + } + } + excl->nra = nra; + srenew(excl->a, excl->nra); + + b2_to_b(b2, excl); +} + +int add_atomtype_decoupled(t_symtab *symtab, gpp_atomtype_t at, + t_nbparam ***nbparam, t_nbparam ***pair) +{ + t_atom atom; + t_param param; + int i, nr; + + /* Define an atom type with all parameters set to zero (no interactions) */ + atom.q = 0.0; + atom.m = 0.0; + /* Type for decoupled atoms could be anything, + * this should be changed automatically later when required. + */ + atom.ptype = eptAtom; + for (i = 0; (i < MAXFORCEPARAM); i++) + { + param.c[i] = 0.0; + } + + nr = add_atomtype(at, symtab, &atom, "decoupled", ¶m, -1, 0.0, 0.0, 0.0, 0, 0, 0); + + /* Add space in the non-bonded parameters matrix */ + realloc_nb_params(at, nbparam, pair); + + return nr; +} + +static void convert_pairs_to_pairsQ(t_params *plist, + real fudgeQQ, t_atoms *atoms) +{ + t_param *paramp1, *paramp2, *paramnew; + int i, j, p1nr, p2nr, p2newnr; + + /* Add the pair list to the pairQ list */ + p1nr = plist[F_LJ14].nr; + p2nr = plist[F_LJC14_Q].nr; + p2newnr = p1nr + p2nr; + snew(paramnew, p2newnr); + + paramp1 = plist[F_LJ14].param; + paramp2 = plist[F_LJC14_Q].param; + + /* Fill in the new F_LJC14_Q array with the old one. NOTE: + it may be possible to just ADD the converted F_LJ14 array + to the old F_LJC14_Q array, but since we have to create + a new sized memory structure, better just to deep copy it all. + */ + + for (i = 0; i < p2nr; i++) + { + /* Copy over parameters */ + for (j = 0; j < 5; j++) /* entries are 0-4 for F_LJC14_Q */ + { + paramnew[i].c[j] = paramp2[i].c[j]; + } + + /* copy over atoms */ + for (j = 0; j < 2; j++) + { + paramnew[i].a[j] = paramp2[i].a[j]; + } + } + + for (i = p2nr; i < p2newnr; i++) + { + j = i-p2nr; + + /* Copy over parameters */ + paramnew[i].c[0] = fudgeQQ; + paramnew[i].c[1] = atoms->atom[paramp1[j].a[0]].q; + paramnew[i].c[2] = atoms->atom[paramp1[j].a[1]].q; + paramnew[i].c[3] = paramp1[j].c[0]; + paramnew[i].c[4] = paramp1[j].c[1]; + + /* copy over atoms */ + paramnew[i].a[0] = paramp1[j].a[0]; + paramnew[i].a[1] = paramp1[j].a[1]; + } + + /* free the old pairlists */ + sfree(plist[F_LJC14_Q].param); + sfree(plist[F_LJ14].param); + + /* now assign the new data to the F_LJC14_Q structure */ + plist[F_LJC14_Q].param = paramnew; + plist[F_LJC14_Q].nr = p2newnr; + + /* Empty the LJ14 pairlist */ + plist[F_LJ14].nr = 0; + plist[F_LJ14].param = NULL; +} + +static void generate_LJCpairsNB(t_molinfo *mol, int nb_funct, t_params *nbp) +{ + int n, ntype, i, j, k; + t_atom *atom; + t_blocka *excl; + gmx_bool bExcl; + t_param param; + + n = mol->atoms.nr; + atom = mol->atoms.atom; + + ntype = static_cast(std::sqrt(static_cast(nbp->nr))); + GMX_ASSERT(ntype * ntype == nbp->nr, "Number of pairs of generated non-bonded parameters should be a perfect square"); + + for (i = 0; i < MAXATOMLIST; i++) + { + param.a[i] = NOTSET; + } + for (i = 0; i < MAXFORCEPARAM; i++) + { + param.c[i] = NOTSET; + } + + /* Add a pair interaction for all non-excluded atom pairs */ + excl = &mol->excls; + for (i = 0; i < n; i++) + { + for (j = i+1; j < n; j++) + { + bExcl = FALSE; + for (k = excl->index[i]; k < excl->index[i+1]; k++) + { + if (excl->a[k] == j) + { + bExcl = TRUE; + } + } + if (!bExcl) + { + if (nb_funct != F_LJ) + { + gmx_fatal(FARGS, "Can only generate non-bonded pair interactions for Van der Waals type Lennard-Jones"); + } + param.a[0] = i; + param.a[1] = j; + param.c[0] = atom[i].q; + param.c[1] = atom[j].q; + param.c[2] = nbp->param[ntype*atom[i].type+atom[j].type].c[0]; + param.c[3] = nbp->param[ntype*atom[i].type+atom[j].type].c[1]; + add_param_to_list(&mol->plist[F_LJC_PAIRS_NB], ¶m); + } + } + } +} + +static void set_excl_all(t_blocka *excl) +{ + int nat, i, j, k; + + /* Get rid of the current exclusions and exclude all atom pairs */ + nat = excl->nr; + excl->nra = nat*nat; + srenew(excl->a, excl->nra); + k = 0; + for (i = 0; i < nat; i++) + { + excl->index[i] = k; + for (j = 0; j < nat; j++) + { + excl->a[k++] = j; + } + } + excl->index[nat] = k; +} + +static void decouple_atoms(t_atoms *atoms, int atomtype_decouple, - int couple_lam0, int couple_lam1) ++ int couple_lam0, int couple_lam1, ++ const char *mol_name) +{ + int i; + + for (i = 0; i < atoms->nr; i++) + { ++ t_atom *atom; ++ ++ atom = &atoms->atom[i]; ++ ++ if (atom->qB != atom->q || atom->typeB != atom->type) ++ { ++ gmx_fatal(FARGS, "Atom %d in molecule type '%s' has different A and B state charges and/or atom types set in the topology file as well as through the mdp option '%s'. You can not use both these methods simultaneously.", ++ i + 1, mol_name, "couple-moltype"); ++ } ++ + if (couple_lam0 == ecouplamNONE || couple_lam0 == ecouplamVDW) + { - atoms->atom[i].q = 0.0; ++ atom->q = 0.0; + } + if (couple_lam0 == ecouplamNONE || couple_lam0 == ecouplamQ) + { - atoms->atom[i].type = atomtype_decouple; ++ atom->type = atomtype_decouple; + } + if (couple_lam1 == ecouplamNONE || couple_lam1 == ecouplamVDW) + { - atoms->atom[i].qB = 0.0; ++ atom->qB = 0.0; + } + if (couple_lam1 == ecouplamNONE || couple_lam1 == ecouplamQ) + { - atoms->atom[i].typeB = atomtype_decouple; ++ atom->typeB = atomtype_decouple; + } + } +} + +void convert_moltype_couple(t_molinfo *mol, int atomtype_decouple, real fudgeQQ, + int couple_lam0, int couple_lam1, + gmx_bool bCoupleIntra, int nb_funct, t_params *nbp) +{ + convert_pairs_to_pairsQ(mol->plist, fudgeQQ, &mol->atoms); + if (!bCoupleIntra) + { + generate_LJCpairsNB(mol, nb_funct, nbp); + set_excl_all(&mol->excls); + } - decouple_atoms(&mol->atoms, atomtype_decouple, couple_lam0, couple_lam1); ++ decouple_atoms(&mol->atoms, atomtype_decouple, couple_lam0, couple_lam1, ++ *mol->name); +} diff --cc src/gromacs/mdlib/calc_verletbuf.cpp index d89960c9fc,0000000000..e8434bec93 mode 100644,000000..100644 --- a/src/gromacs/mdlib/calc_verletbuf.cpp +++ b/src/gromacs/mdlib/calc_verletbuf.cpp @@@ -1,1120 -1,0 +1,1129 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * - * Copyright (c) 2012,2013,2014,2015, by the GROMACS development team, led by ++ * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ +#include "gmxpre.h" + +#include "calc_verletbuf.h" + +#include +#include + +#include + +#include + +#include "gromacs/math/calculate-ewald-splitting-coefficient.h" +#include "gromacs/math/functions.h" +#include "gromacs/math/units.h" +#include "gromacs/math/vec.h" +#include "gromacs/mdlib/nb_verlet.h" +#include "gromacs/mdlib/nbnxn_simd.h" +#include "gromacs/mdlib/nbnxn_util.h" +#include "gromacs/mdtypes/inputrec.h" +#include "gromacs/mdtypes/md_enums.h" +#include "gromacs/topology/ifunc.h" +#include "gromacs/topology/topology.h" +#include "gromacs/utility/fatalerror.h" +#include "gromacs/utility/smalloc.h" + +/* The code in this file estimates a pairlist buffer length + * given a target energy drift per atom per picosecond. + * This is done by estimating the drift given a buffer length. + * Ideally we would like to have a tight overestimate of the drift, + * but that can be difficult to achieve. + * + * Significant approximations used: + * + * Uniform particle density. UNDERESTIMATES the drift by rho_global/rho_local. + * + * Interactions don't affect particle motion. OVERESTIMATES the drift on longer + * time scales. This approximation probably introduces the largest errors. + * + * Only take one constraint per particle into account: OVERESTIMATES the drift. + * + * For rotating constraints assume the same functional shape for time scales + * where the constraints rotate significantly as the exact expression for + * short time scales. OVERESTIMATES the drift on long time scales. + * + * For non-linear virtual sites use the mass of the lightest constructing atom + * to determine the displacement. OVER/UNDERESTIMATES the drift, depending on + * the geometry and masses of constructing atoms. + * + * Note that the formulas for normal atoms and linear virtual sites are exact, + * apart from the first two approximations. + * + * Note that apart from the effect of the above approximations, the actual + * drift of the total energy of a system can be order of magnitude smaller + * due to cancellation of positive and negative drift for different pairs. + */ + + +/* Struct for unique atom type for calculating the energy drift. + * The atom displacement depends on mass and constraints. + * The energy jump for given distance depend on LJ type and q. + */ +typedef struct +{ + real mass; /* mass */ + int type; /* type (used for LJ parameters) */ + real q; /* charge */ + gmx_bool bConstr; /* constrained, if TRUE, use #DOF=2 iso 3 */ + real con_mass; /* mass of heaviest atom connected by constraints */ + real con_len; /* constraint length to the heaviest atom */ +} atom_nonbonded_kinetic_prop_t; + +/* Struct for unique atom type for calculating the energy drift. + * The atom displacement depends on mass and constraints. + * The energy jump for given distance depend on LJ type and q. + */ +typedef struct +{ + atom_nonbonded_kinetic_prop_t prop; /* non-bonded and kinetic atom prop. */ + int n; /* #atoms of this type in the system */ +} verletbuf_atomtype_t; + +void verletbuf_get_list_setup(gmx_bool gmx_unused bSIMD, + gmx_bool bGPU, + verletbuf_list_setup_t *list_setup) +{ + /* When calling this function we often don't know which kernel type we + * are going to use. W choose the kernel type with the smallest possible + * i- and j-cluster sizes, so we potentially overestimate, but never + * underestimate, the buffer drift. + * Note that the current buffer estimation code only handles clusters + * of size 1, 2 or 4, so for 4x8 or 8x8 we use the estimate for 4x4. + */ + + if (bGPU) + { + /* The CUDA kernels split the j-clusters in two halves */ + list_setup->cluster_size_i = nbnxn_kernel_to_cluster_i_size(nbnxnk8x8x8_GPU); + list_setup->cluster_size_j = nbnxn_kernel_to_cluster_j_size(nbnxnk8x8x8_GPU)/2; + } + else + { + int kernel_type; + + kernel_type = nbnxnk4x4_PlainC; + +#if GMX_SIMD + if (bSIMD) + { +#ifdef GMX_NBNXN_SIMD_2XNN + /* We use the smallest cluster size to be on the safe side */ + kernel_type = nbnxnk4xN_SIMD_2xNN; +#else + kernel_type = nbnxnk4xN_SIMD_4xN; +#endif + } +#endif + + list_setup->cluster_size_i = nbnxn_kernel_to_cluster_i_size(kernel_type); + list_setup->cluster_size_j = nbnxn_kernel_to_cluster_j_size(kernel_type); + } +} + +static gmx_bool +atom_nonbonded_kinetic_prop_equal(const atom_nonbonded_kinetic_prop_t *prop1, + const atom_nonbonded_kinetic_prop_t *prop2) +{ + return (prop1->mass == prop2->mass && + prop1->type == prop2->type && + prop1->q == prop2->q && + prop1->bConstr == prop2->bConstr && + prop1->con_mass == prop2->con_mass && + prop1->con_len == prop2->con_len); +} + +static void add_at(verletbuf_atomtype_t **att_p, int *natt_p, + const atom_nonbonded_kinetic_prop_t *prop, + int nmol) +{ + verletbuf_atomtype_t *att; + int natt, i; + + if (prop->mass == 0) + { + /* Ignore massless particles */ + return; + } + + att = *att_p; + natt = *natt_p; + + i = 0; + while (i < natt && !atom_nonbonded_kinetic_prop_equal(prop, &att[i].prop)) + { + i++; + } + + if (i < natt) + { + att[i].n += nmol; + } + else + { + (*natt_p)++; + srenew(*att_p, *natt_p); + (*att_p)[i].prop = *prop; + (*att_p)[i].n = nmol; + } +} + +static void get_vsite_masses(const gmx_moltype_t *moltype, + const gmx_ffparams_t *ffparams, + real *vsite_m, + int *n_nonlin_vsite) +{ + int ft, i; + const t_ilist *il; + + *n_nonlin_vsite = 0; + + /* Check for virtual sites, determine mass from constructing atoms */ + for (ft = 0; ft < F_NRE; ft++) + { + if (IS_VSITE(ft)) + { + il = &moltype->ilist[ft]; + + for (i = 0; i < il->nr; i += 1+NRAL(ft)) + { + const t_iparams *ip; + real inv_mass, coeff, m_aj; + int a1, aj; + + ip = &ffparams->iparams[il->iatoms[i]]; + + a1 = il->iatoms[i+1]; + + if (ft != F_VSITEN) + { + /* Only vsiten can have more than four + constructing atoms, so NRAL(ft) <= 5 */ + int j; + real *cam; + const int maxj = NRAL(ft); + + snew(cam, maxj); + assert(maxj <= 5); + for (j = 1; j < maxj; j++) + { + cam[j] = moltype->atoms.atom[il->iatoms[i+1+j]].m; + if (cam[j] == 0) + { + cam[j] = vsite_m[il->iatoms[i+1+j]]; + } + if (cam[j] == 0) + { + gmx_fatal(FARGS, "In molecule type '%s' %s construction involves atom %d, which is a virtual site of equal or high complexity. This is not supported.", + *moltype->name, + interaction_function[ft].longname, + il->iatoms[i+1+j]+1); + } + } + + switch (ft) + { + case F_VSITE2: + /* Exact */ + vsite_m[a1] = (cam[1]*cam[2])/(cam[2]*gmx::square(1-ip->vsite.a) + cam[1]*gmx::square(ip->vsite.a)); + break; + case F_VSITE3: + /* Exact */ + vsite_m[a1] = (cam[1]*cam[2]*cam[3])/(cam[2]*cam[3]*gmx::square(1-ip->vsite.a-ip->vsite.b) + cam[1]*cam[3]*gmx::square(ip->vsite.a) + cam[1]*cam[2]*gmx::square(ip->vsite.b)); + break; + case F_VSITEN: + gmx_incons("Invalid vsite type"); + break; + default: + /* Use the mass of the lightest constructing atom. + * This is an approximation. + * If the distance of the virtual site to the + * constructing atom is less than all distances + * between constructing atoms, this is a safe + * over-estimate of the displacement of the vsite. + * This condition holds for all H mass replacement + * vsite constructions, except for SP2/3 groups. + * In SP3 groups one H will have a F_VSITE3 + * construction, so even there the total drift + * estimate shouldn't be far off. + */ + vsite_m[a1] = cam[1]; + for (j = 2; j < maxj; j++) + { + vsite_m[a1] = std::min(vsite_m[a1], cam[j]); + } + (*n_nonlin_vsite)++; + break; + } + sfree(cam); + } + else + { + int j; + + /* Exact */ + inv_mass = 0; + for (j = 0; j < 3*ffparams->iparams[il->iatoms[i]].vsiten.n; j += 3) + { + aj = il->iatoms[i+j+2]; + coeff = ffparams->iparams[il->iatoms[i+j]].vsiten.a; + if (moltype->atoms.atom[aj].ptype == eptVSite) + { + m_aj = vsite_m[aj]; + } + else + { + m_aj = moltype->atoms.atom[aj].m; + } + if (m_aj <= 0) + { + gmx_incons("The mass of a vsiten constructing atom is <= 0"); + } + inv_mass += coeff*coeff/m_aj; + } + vsite_m[a1] = 1/inv_mass; + /* Correct for loop increment of i */ + i += j - 1 - NRAL(ft); + } + if (gmx_debug_at) + { + fprintf(debug, "atom %4d %-20s mass %6.3f\n", + a1, interaction_function[ft].longname, vsite_m[a1]); + } + } + } + } +} + +static void get_verlet_buffer_atomtypes(const gmx_mtop_t *mtop, + verletbuf_atomtype_t **att_p, + int *natt_p, + int *n_nonlin_vsite) +{ + verletbuf_atomtype_t *att; + int natt; + int mb, nmol, ft, i, a1, a2, a3, a; + const t_atoms *atoms; + const t_ilist *il; + const t_iparams *ip; + atom_nonbonded_kinetic_prop_t *prop; + real *vsite_m; + int n_nonlin_vsite_mol; + + att = NULL; + natt = 0; + + if (n_nonlin_vsite != NULL) + { + *n_nonlin_vsite = 0; + } + + for (mb = 0; mb < mtop->nmolblock; mb++) + { + nmol = mtop->molblock[mb].nmol; + + atoms = &mtop->moltype[mtop->molblock[mb].type].atoms; + + /* Check for constraints, as they affect the kinetic energy. + * For virtual sites we need the masses and geometry of + * the constructing atoms to determine their velocity distribution. + */ + snew(prop, atoms->nr); + snew(vsite_m, atoms->nr); + + for (ft = F_CONSTR; ft <= F_CONSTRNC; ft++) + { + il = &mtop->moltype[mtop->molblock[mb].type].ilist[ft]; + + for (i = 0; i < il->nr; i += 1+NRAL(ft)) + { + ip = &mtop->ffparams.iparams[il->iatoms[i]]; + a1 = il->iatoms[i+1]; + a2 = il->iatoms[i+2]; + if (atoms->atom[a2].m > prop[a1].con_mass) + { + prop[a1].con_mass = atoms->atom[a2].m; + prop[a1].con_len = ip->constr.dA; + } + if (atoms->atom[a1].m > prop[a2].con_mass) + { + prop[a2].con_mass = atoms->atom[a1].m; + prop[a2].con_len = ip->constr.dA; + } + } + } + + il = &mtop->moltype[mtop->molblock[mb].type].ilist[F_SETTLE]; + + for (i = 0; i < il->nr; i += 1+NRAL(F_SETTLE)) + { + ip = &mtop->ffparams.iparams[il->iatoms[i]]; + a1 = il->iatoms[i+1]; + a2 = il->iatoms[i+2]; + a3 = il->iatoms[i+3]; + /* Usually the mass of a1 (usually oxygen) is larger than a2/a3. + * If this is not the case, we overestimate the displacement, + * which leads to a larger buffer (ok since this is an exotic case). + */ + prop[a1].con_mass = atoms->atom[a2].m; + prop[a1].con_len = ip->settle.doh; + + prop[a2].con_mass = atoms->atom[a1].m; + prop[a2].con_len = ip->settle.doh; + + prop[a3].con_mass = atoms->atom[a1].m; + prop[a3].con_len = ip->settle.doh; + } + + get_vsite_masses(&mtop->moltype[mtop->molblock[mb].type], + &mtop->ffparams, + vsite_m, + &n_nonlin_vsite_mol); + if (n_nonlin_vsite != NULL) + { + *n_nonlin_vsite += nmol*n_nonlin_vsite_mol; + } + + for (a = 0; a < atoms->nr; a++) + { + if (atoms->atom[a].ptype == eptVSite) + { + prop[a].mass = vsite_m[a]; + } + else + { + prop[a].mass = atoms->atom[a].m; + } + prop[a].type = atoms->atom[a].type; + prop[a].q = atoms->atom[a].q; + /* We consider an atom constrained, #DOF=2, when it is + * connected with constraints to (at least one) atom with + * a mass of more than 0.4x its own mass. This is not a critical + * parameter, since with roughly equal masses the unconstrained + * and constrained displacement will not differ much (and both + * overestimate the displacement). + */ + prop[a].bConstr = (prop[a].con_mass > 0.4*prop[a].mass); + + add_at(&att, &natt, &prop[a], nmol); + } + + sfree(vsite_m); + sfree(prop); + } + + if (gmx_debug_at) + { + for (a = 0; a < natt; a++) + { + fprintf(debug, "type %d: m %5.2f t %d q %6.3f con %d con_m %5.3f con_l %5.3f n %d\n", + a, att[a].prop.mass, att[a].prop.type, att[a].prop.q, + att[a].prop.bConstr, att[a].prop.con_mass, att[a].prop.con_len, + att[a].n); + } + } + + *att_p = att; + *natt_p = natt; +} + +/* This function computes two components of the estimate of the variance + * in the displacement of one atom in a system of two constrained atoms. + * Returns in sigma2_2d the variance due to rotation of the constrained + * atom around the atom to which it constrained. + * Returns in sigma2_3d the variance due to displacement of the COM + * of the whole system of the two constrained atoms. + * + * Note that we only take a single constraint (the one to the heaviest atom) + * into account. If an atom has multiple constraints, this will result in + * an overestimate of the displacement, which gives a larger drift and buffer. + */ +static void constrained_atom_sigma2(real kT_fac, + const atom_nonbonded_kinetic_prop_t *prop, + real *sigma2_2d, + real *sigma2_3d) +{ + real sigma2_rot; + real com_dist; + real sigma2_rel; + real scale; + + /* Here we decompose the motion of a constrained atom into two + * components: rotation around the COM and translation of the COM. + */ + + /* Determine the variance for the displacement of the rotational mode */ + sigma2_rot = kT_fac/(prop->mass*(prop->mass + prop->con_mass)/prop->con_mass); + + /* The distance from the atom to the COM, i.e. the rotational arm */ + com_dist = prop->con_len*prop->con_mass/(prop->mass + prop->con_mass); + + /* The variance relative to the arm */ + sigma2_rel = sigma2_rot/(com_dist*com_dist); + /* At 6 the scaling formula has slope 0, + * so we keep sigma2_2d constant after that. + */ + if (sigma2_rel < 6) + { + /* A constrained atom rotates around the atom it is constrained to. + * This results in a smaller linear displacement than for a free atom. + * For a perfectly circular displacement, this lowers the displacement + * by: 1/arcsin(arc_length) + * and arcsin(x) = 1 + x^2/6 + ... + * For sigma2_rel<<1 the displacement distribution is erfc + * (exact formula is provided below). For larger sigma, it is clear + * that the displacement can't be larger than 2*com_dist. + * It turns out that the distribution becomes nearly uniform. + * For intermediate sigma2_rel, scaling down sigma with the third + * order expansion of arcsin with argument sigma_rel turns out + * to give a very good approximation of the distribution and variance. + * Even for larger values, the variance is only slightly overestimated. + * Note that the most relevant displacements are in the long tail. + * This rotation approximation always overestimates the tail (which + * runs to infinity, whereas it should be <= 2*com_dist). + * Thus we always overestimate the drift and the buffer size. + */ + scale = 1/(1 + sigma2_rel/6); + *sigma2_2d = sigma2_rot*scale*scale; + } + else + { + /* sigma_2d is set to the maximum given by the scaling above. + * For large sigma2 the real displacement distribution is close + * to uniform over -2*con_len to 2*com_dist. + * Our erfc with sigma_2d=sqrt(1.5)*com_dist (which means the sigma + * of the erfc output distribution is con_dist) overestimates + * the variance and additionally has a long tail. This means + * we have a (safe) overestimation of the drift. + */ + *sigma2_2d = 1.5*com_dist*com_dist; + } + + /* The constrained atom also moves (in 3D) with the COM of both atoms */ + *sigma2_3d = kT_fac/(prop->mass + prop->con_mass); +} + +static void get_atom_sigma2(real kT_fac, + const atom_nonbonded_kinetic_prop_t *prop, + real *sigma2_2d, + real *sigma2_3d) +{ + if (prop->bConstr) + { + /* Complicated constraint calculation in a separate function */ + constrained_atom_sigma2(kT_fac, prop, sigma2_2d, sigma2_3d); + } + else + { + /* Unconstrained atom: trivial */ + *sigma2_2d = 0; + *sigma2_3d = kT_fac/prop->mass; + } +} + +static void approx_2dof(real s2, real x, real *shift, real *scale) +{ + /* A particle with 1 DOF constrained has 2 DOFs instead of 3. + * This code is also used for particles with multiple constraints, + * in which case we overestimate the displacement. + * The 2DOF distribution is sqrt(pi/2)*erfc(r/(sqrt(2)*s))/(2*s). + * We approximate this with scale*Gaussian(s,r+shift), + * by matching the distribution value and derivative at x. + * This is a tight overestimate for all r>=0 at any s and x. + */ + real ex, er; + + ex = exp(-x*x/(2*s2)); + er = std::erfc(x/std::sqrt(2*s2)); + + *shift = -x + std::sqrt(2*s2/M_PI)*ex/er; + *scale = 0.5*M_PI*std::exp(ex*ex/(M_PI*er*er))*er; +} + +static real ener_drift(const verletbuf_atomtype_t *att, int natt, + const gmx_ffparams_t *ffp, + real kT_fac, + real md1_ljd, real d2_ljd, real md3_ljd, + real md1_ljr, real d2_ljr, real md3_ljr, + real md1_el, real d2_el, + real r_buffer, + real rlist, real boxvol) +{ + /* Erfc(8)=1e-29, use this limit so we have some space for arithmetic + * on the result when using float precision. + */ + const real erfc_arg_max = 8.0; + + double drift_tot, pot1, pot2, pot3, pot; + int i, j; + real s2i_2d, s2i_3d, s2j_2d, s2j_3d, s2, s; + int ti, tj; + real md1, d2, md3; + real sc_fac, rsh, rsh2; + double c_exp, c_erfc; + + drift_tot = 0; + + /* Loop over the different atom type pairs */ + for (i = 0; i < natt; i++) + { + get_atom_sigma2(kT_fac, &att[i].prop, &s2i_2d, &s2i_3d); + ti = att[i].prop.type; + + for (j = i; j < natt; j++) + { + get_atom_sigma2(kT_fac, &att[j].prop, &s2j_2d, &s2j_3d); + tj = att[j].prop.type; + + /* Add up the up to four independent variances */ + s2 = s2i_2d + s2i_3d + s2j_2d + s2j_3d; + + /* Note that attractive and repulsive potentials for individual + * pairs will partially cancel. + */ + /* -dV/dr at the cut-off for LJ + Coulomb */ + md1 = + md1_ljd*ffp->iparams[ti*ffp->atnr+tj].lj.c6 + + md1_ljr*ffp->iparams[ti*ffp->atnr+tj].lj.c12 + + md1_el*att[i].prop.q*att[j].prop.q; + + /* d2V/dr2 at the cut-off for LJ + Coulomb */ + d2 = + d2_ljd*ffp->iparams[ti*ffp->atnr+tj].lj.c6 + + d2_ljr*ffp->iparams[ti*ffp->atnr+tj].lj.c12 + + d2_el*att[i].prop.q*att[j].prop.q; + + /* -d3V/dr3 at the cut-off for LJ, we neglect Coulomb */ + md3 = + md3_ljd*ffp->iparams[ti*ffp->atnr+tj].lj.c6 + + md3_ljr*ffp->iparams[ti*ffp->atnr+tj].lj.c12; + + rsh = r_buffer; + sc_fac = 1.0; + + if (rsh*rsh > 2*s2*erfc_arg_max*erfc_arg_max) + { + /* Erfc might run out of float and become 0, somewhat before + * c_exp becomes 0. To avoid this and to avoid NaN in + * approx_2dof, we set both c_expc and c_erfc to zero. + * In any relevant case this has no effect on the results, + * since c_exp < 6e-29, so the displacement is completely + * negligible for such atom pairs (and an overestimate). + * In nearly all use cases, there will be other atom + * pairs that contribute much more to the total, so zeroing + * this particular contribution has no effect at all. + */ + c_exp = 0; + c_erfc = 0; + } + else + { + /* For constraints: adapt r and scaling for the Gaussian */ + if (att[i].prop.bConstr) + { + real sh, sc; + + approx_2dof(s2i_2d, r_buffer*s2i_2d/s2, &sh, &sc); + rsh += sh; + sc_fac *= sc; + } + if (att[j].prop.bConstr) + { + real sh, sc; + + approx_2dof(s2j_2d, r_buffer*s2j_2d/s2, &sh, &sc); + rsh += sh; + sc_fac *= sc; + } + + /* Exact contribution of an atom pair with Gaussian displacement + * with sigma s to the energy drift for a potential with + * derivative -md and second derivative dd at the cut-off. + * The only catch is that for potentials that change sign + * near the cut-off there could be an unlucky compensation + * of positive and negative energy drift. + * Such potentials are extremely rare though. + * + * Note that pot has unit energy*length, as the linear + * atom density still needs to be put in. + */ + c_exp = std::exp(-rsh*rsh/(2*s2))/std::sqrt(2*M_PI); + c_erfc = 0.5*std::erfc(rsh/(std::sqrt(2*s2))); + } + s = std::sqrt(s2); + rsh2 = rsh*rsh; + + pot1 = sc_fac* + md1/2*((rsh2 + s2)*c_erfc - rsh*s*c_exp); + pot2 = sc_fac* + d2/6*(s*(rsh2 + 2*s2)*c_exp - rsh*(rsh2 + 3*s2)*c_erfc); + pot3 = sc_fac* + md3/24*((rsh2*rsh2 + 6*rsh2*s2 + 3*s2*s2)*c_erfc - rsh*s*(rsh2 + 5*s2)*c_exp); + pot = pot1 + pot2 + pot3; + + if (gmx_debug_at) + { + fprintf(debug, "n %d %d d s %.3f %.3f %.3f %.3f con %d -d1 %8.1e d2 %8.1e -d3 %8.1e pot1 %8.1e pot2 %8.1e pot3 %8.1e pot %8.1e\n", + att[i].n, att[j].n, + std::sqrt(s2i_2d), std::sqrt(s2i_3d), + std::sqrt(s2j_2d), std::sqrt(s2j_3d), + att[i].prop.bConstr+att[j].prop.bConstr, + md1, d2, md3, + pot1, pot2, pot3, pot); + } + + /* Multiply by the number of atom pairs */ + if (j == i) + { + pot *= (double)att[i].n*(att[i].n - 1)/2; + } + else + { + pot *= (double)att[i].n*att[j].n; + } + /* We need the line density to get the energy drift of the system. + * The effective average r^2 is close to (rlist+sigma)^2. + */ + pot *= 4*M_PI*gmx::square(rlist + s)/boxvol; + + /* Add the unsigned drift to avoid cancellation of errors */ + drift_tot += std::abs(pot); + } + } + + return drift_tot; +} + +static real surface_frac(int cluster_size, real particle_distance, real rlist) +{ + real d, area_rel; + + if (rlist < 0.5*particle_distance) + { + /* We have non overlapping spheres */ + return 1.0; + } + + /* Half the inter-particle distance relative to rlist */ + d = 0.5*particle_distance/rlist; + + /* Determine the area of the surface at distance rlist to the closest + * particle, relative to surface of a sphere of radius rlist. + * The formulas below assume close to cubic cells for the pair search grid, + * which the pair search code tries to achieve. + * Note that in practice particle distances will not be delta distributed, + * but have some spread, often involving shorter distances, + * as e.g. O-H bonds in a water molecule. Thus the estimates below will + * usually be slightly too high and thus conservative. + */ + switch (cluster_size) + { + case 1: + /* One particle: trivial */ + area_rel = 1.0; + break; + case 2: + /* Two particles: two spheres at fractional distance 2*a */ + area_rel = 1.0 + d; + break; + case 4: + /* We assume a perfect, symmetric tetrahedron geometry. + * The surface around a tetrahedron is too complex for a full + * analytical solution, so we use a Taylor expansion. + */ + area_rel = (1.0 + 1/M_PI*(6*std::acos(1/std::sqrt(3))*d + + std::sqrt(3)*d*d*(1.0 + + 5.0/18.0*d*d + + 7.0/45.0*d*d*d*d + + 83.0/756.0*d*d*d*d*d*d))); + break; + default: + gmx_incons("surface_frac called with unsupported cluster_size"); + area_rel = 1.0; + } + + return area_rel/cluster_size; +} + +/* Returns the negative of the third derivative of a potential r^-p + * with a force-switch function, evaluated at the cut-off rc. + */ +static real md3_force_switch(real p, real rswitch, real rc) +{ + /* The switched force function is: + * p*r^-(p+1) + a*(r - rswitch)^2 + b*(r - rswitch)^3 + */ + real a, b; + real md3_pot, md3_sw; + + a = -((p + 4)*rc - (p + 1)*rswitch)/(pow(rc, p+2)*gmx::square(rc-rswitch)); + b = ((p + 3)*rc - (p + 1)*rswitch)/(pow(rc, p+2)*gmx::power3(rc-rswitch)); + + md3_pot = (p + 2)*(p + 1)*p*pow(rc, p+3); + md3_sw = 2*a + 6*b*(rc - rswitch); + + return md3_pot + md3_sw; +} + +void calc_verlet_buffer_size(const gmx_mtop_t *mtop, real boxvol, + const t_inputrec *ir, + real reference_temperature, + const verletbuf_list_setup_t *list_setup, + int *n_nonlin_vsite, + real *rlist) +{ + double resolution; + char *env; + + real particle_distance; + real nb_clust_frac_pairs_not_in_list_at_cutoff; + + verletbuf_atomtype_t *att = NULL; + int natt = -1, i; + double reppow; + real md1_ljd, d2_ljd, md3_ljd; + real md1_ljr, d2_ljr, md3_ljr; + real md1_el, d2_el; + real elfac; + real kT_fac, mass_min; + int ib0, ib1, ib; + real rb, rl; + real drift; + ++ if (!EI_DYNAMICS(ir->eI)) ++ { ++ gmx_incons("Can only determine the Verlet buffer size for integrators that perform dynamics"); ++ } ++ if (ir->verletbuf_tol <= 0) ++ { ++ gmx_incons("The Verlet buffer tolerance needs to be larger than zero"); ++ } ++ + if (reference_temperature < 0) + { + if (EI_MD(ir->eI) && ir->etc == etcNO) + { + /* This case should be handled outside calc_verlet_buffer_size */ + gmx_incons("calc_verlet_buffer_size called with an NVE ensemble and reference_temperature < 0"); + } + + /* We use the maximum temperature with multiple T-coupl groups. + * We could use a per particle temperature, but since particles + * interact, this might underestimate the buffer size. + */ + reference_temperature = 0; + for (i = 0; i < ir->opts.ngtc; i++) + { + if (ir->opts.tau_t[i] >= 0) + { + reference_temperature = std::max(reference_temperature, + ir->opts.ref_t[i]); + } + } + } + + /* Resolution of the buffer size */ + resolution = 0.001; + + env = getenv("GMX_VERLET_BUFFER_RES"); + if (env != NULL) + { + sscanf(env, "%lf", &resolution); + } + + /* In an atom wise pair-list there would be no pairs in the list + * beyond the pair-list cut-off. + * However, we use a pair-list of groups vs groups of atoms. + * For groups of 4 atoms, the parallelism of SSE instructions, only + * 10% of the atoms pairs are not in the list just beyond the cut-off. + * As this percentage increases slowly compared to the decrease of the + * Gaussian displacement distribution over this range, we can simply + * reduce the drift by this fraction. + * For larger groups, e.g. of 8 atoms, this fraction will be lower, + * so then buffer size will be on the conservative (large) side. + * + * Note that the formulas used here do not take into account + * cancellation of errors which could occur by missing both + * attractive and repulsive interactions. + * + * The only major assumption is homogeneous particle distribution. + * For an inhomogeneous system, such as a liquid-vapor system, + * the buffer will be underestimated. The actual energy drift + * will be higher by the factor: local/homogeneous particle density. + * + * The results of this estimate have been checked againt simulations. + * In most cases the real drift differs by less than a factor 2. + */ + + /* Worst case assumption: HCP packing of particles gives largest distance */ + particle_distance = std::cbrt(boxvol*std::sqrt(2)/mtop->natoms); + + get_verlet_buffer_atomtypes(mtop, &att, &natt, n_nonlin_vsite); + assert(att != NULL && natt >= 0); + + if (debug) + { + fprintf(debug, "particle distance assuming HCP packing: %f nm\n", + particle_distance); + fprintf(debug, "energy drift atom types: %d\n", natt); + } + + reppow = mtop->ffparams.reppow; + md1_ljd = 0; + d2_ljd = 0; + md3_ljd = 0; + md1_ljr = 0; + d2_ljr = 0; + md3_ljr = 0; + if (ir->vdwtype == evdwCUT) + { + real sw_range, md3_pswf; + + switch (ir->vdw_modifier) + { + case eintmodNONE: + case eintmodPOTSHIFT: + /* -dV/dr of -r^-6 and r^-reppow */ + md1_ljd = -6/(ir->rvdw*gmx::power6(ir->rvdw)); + md1_ljr = reppow*pow(ir->rvdw, -(reppow+1)); + /* The contribution of the higher derivatives is negligible */ + break; + case eintmodFORCESWITCH: + /* At the cut-off: V=V'=V''=0, so we use only V''' */ + md3_ljd = -md3_force_switch(6.0, ir->rvdw_switch, ir->rvdw); + md3_ljr = md3_force_switch(reppow, ir->rvdw_switch, ir->rvdw); + break; + case eintmodPOTSWITCH: + /* At the cut-off: V=V'=V''=0. + * V''' is given by the original potential times + * the third derivative of the switch function. + */ + sw_range = ir->rvdw - ir->rvdw_switch; + md3_pswf = 60.0/gmx::power3(sw_range); + + md3_ljd = -1.0/gmx::power6(ir->rvdw)*md3_pswf; + md3_ljr = pow(ir->rvdw, -reppow)*md3_pswf; + break; + default: + gmx_incons("Unimplemented VdW modifier"); + } + } + else if (EVDW_PME(ir->vdwtype)) + { + real b, r, br, br2, br4, br6; + b = calc_ewaldcoeff_lj(ir->rvdw, ir->ewald_rtol_lj); + r = ir->rvdw; + br = b*r; + br2 = br*br; + br4 = br2*br2; + br6 = br4*br2; + /* -dV/dr of g(br)*r^-6 [where g(x) = exp(-x^2)(1+x^2+x^4/2), see LJ-PME equations in manual] and r^-reppow */ + md1_ljd = -std::exp(-br2)*(br6 + 3.0*br4 + 6.0*br2 + 6.0)/(r*gmx::power6(r)); + md1_ljr = reppow*std::pow(r, -(reppow+1)); + /* The contribution of the higher derivatives is negligible */ + } + else + { + gmx_fatal(FARGS, "Energy drift calculation is only implemented for plain cut-off Lennard-Jones interactions"); + } + + elfac = ONE_4PI_EPS0/ir->epsilon_r; + + /* Determine md=-dV/dr and dd=d^2V/dr^2 */ + md1_el = 0; + if (ir->coulombtype == eelCUT || EEL_RF(ir->coulombtype)) + { + real eps_rf, k_rf; + + if (ir->coulombtype == eelCUT) + { + eps_rf = 1; + k_rf = 0; + } + else + { + eps_rf = ir->epsilon_rf/ir->epsilon_r; + if (eps_rf != 0) + { + k_rf = (eps_rf - ir->epsilon_r)/( gmx::power3(ir->rcoulomb) * (2*eps_rf + ir->epsilon_r) ); + } + else + { + /* epsilon_rf = infinity */ + k_rf = 0.5/gmx::power3(ir->rcoulomb); + } + } + + if (eps_rf > 0) + { + md1_el = elfac*(1.0/gmx::square(ir->rcoulomb) - 2*k_rf*ir->rcoulomb); + } + d2_el = elfac*(2.0/gmx::power3(ir->rcoulomb) + 2*k_rf); + } + else if (EEL_PME(ir->coulombtype) || ir->coulombtype == eelEWALD) + { + real b, rc, br; + + b = calc_ewaldcoeff_q(ir->rcoulomb, ir->ewald_rtol); + rc = ir->rcoulomb; + br = b*rc; + md1_el = elfac*(b*exp(-br*br)*M_2_SQRTPI/rc + std::erfc(br)/(rc*rc)); + d2_el = elfac/(rc*rc)*(2*b*(1 + br*br)*exp(-br*br)*M_2_SQRTPI + 2*std::erfc(br)/rc); + } + else + { + gmx_fatal(FARGS, "Energy drift calculation is only implemented for Reaction-Field and Ewald electrostatics"); + } + + /* Determine the variance of the atomic displacement + * over nstlist-1 steps: kT_fac + * For inertial dynamics (not Brownian dynamics) the mass factor + * is not included in kT_fac, it is added later. + */ + if (ir->eI == eiBD) + { + /* Get the displacement distribution from the random component only. + * With accurate integration the systematic (force) displacement + * should be negligible (unless nstlist is extremely large, which + * you wouldn't do anyhow). + */ + kT_fac = 2*BOLTZ*reference_temperature*(ir->nstlist-1)*ir->delta_t; + if (ir->bd_fric > 0) + { + /* This is directly sigma^2 of the displacement */ + kT_fac /= ir->bd_fric; + + /* Set the masses to 1 as kT_fac is the full sigma^2, + * but we divide by m in ener_drift(). + */ + for (i = 0; i < natt; i++) + { + att[i].prop.mass = 1; + } + } + else + { + real tau_t; + + /* Per group tau_t is not implemented yet, use the maximum */ + tau_t = ir->opts.tau_t[0]; + for (i = 1; i < ir->opts.ngtc; i++) + { + tau_t = std::max(tau_t, ir->opts.tau_t[i]); + } + + kT_fac *= tau_t; + /* This kT_fac needs to be divided by the mass to get sigma^2 */ + } + } + else + { + kT_fac = BOLTZ*reference_temperature*gmx::square((ir->nstlist-1)*ir->delta_t); + } + + mass_min = att[0].prop.mass; + for (i = 1; i < natt; i++) + { + mass_min = std::min(mass_min, att[i].prop.mass); + } + + if (debug) + { + fprintf(debug, "md1_ljd %9.2e d2_ljd %9.2e md3_ljd %9.2e\n", md1_ljd, d2_ljd, md3_ljd); + fprintf(debug, "md1_ljr %9.2e d2_ljr %9.2e md3_ljr %9.2e\n", md1_ljr, d2_ljr, md3_ljr); + fprintf(debug, "md1_el %9.2e d2_el %9.2e\n", md1_el, d2_el); + fprintf(debug, "sqrt(kT_fac) %f\n", std::sqrt(kT_fac)); + fprintf(debug, "mass_min %f\n", mass_min); + } + + /* Search using bisection */ + ib0 = -1; + /* The drift will be neglible at 5 times the max sigma */ + ib1 = (int)(5*2*std::sqrt(kT_fac/mass_min)/resolution) + 1; + while (ib1 - ib0 > 1) + { + ib = (ib0 + ib1)/2; + rb = ib*resolution; + rl = std::max(ir->rvdw, ir->rcoulomb) + rb; + + /* Calculate the average energy drift at the last step + * of the nstlist steps at which the pair-list is used. + */ + drift = ener_drift(att, natt, &mtop->ffparams, + kT_fac, + md1_ljd, d2_ljd, md3_ljd, + md1_ljr, d2_ljr, md3_ljr, + md1_el, d2_el, + rb, + rl, boxvol); + + /* Correct for the fact that we are using a Ni x Nj particle pair list + * and not a 1 x 1 particle pair list. This reduces the drift. + */ + /* We don't have a formula for 8 (yet), use 4 which is conservative */ + nb_clust_frac_pairs_not_in_list_at_cutoff = + surface_frac(std::min(list_setup->cluster_size_i, 4), + particle_distance, rl)* + surface_frac(std::min(list_setup->cluster_size_j, 4), + particle_distance, rl); + drift *= nb_clust_frac_pairs_not_in_list_at_cutoff; + + /* Convert the drift to drift per unit time per atom */ + drift /= ir->nstlist*ir->delta_t*mtop->natoms; + + if (debug) + { + fprintf(debug, "ib %3d %3d %3d rb %.3f %dx%d fac %.3f drift %.1e\n", + ib0, ib, ib1, rb, + list_setup->cluster_size_i, list_setup->cluster_size_j, + nb_clust_frac_pairs_not_in_list_at_cutoff, + drift); + } + + if (std::abs(drift) > ir->verletbuf_tol) + { + ib0 = ib; + } + else + { + ib1 = ib; + } + } + + sfree(att); + + *rlist = std::max(ir->rvdw, ir->rcoulomb) + ib1*resolution; +} diff --cc src/gromacs/mdlib/clincs.cpp index bf9235138b,fcd8de942d..bc8bbf91ea --- a/src/gromacs/mdlib/clincs.cpp +++ b/src/gromacs/mdlib/clincs.cpp @@@ -1253,16 -1470,13 +1269,17 @@@ void set_lincs_matrix(struct gmx_lincsd } /* Construct the coupling coefficient matrix blmf */ - int th, ntriangle = 0, ncc_triangle = 0; - #pragma omp parallel for reduction(+: ntriangle, ncc_triangle) num_threads(li->ntask) schedule(static) + int th, ntriangle = 0, ncc_triangle = 0, nCrossTaskTriangles = 0; + #pragma omp parallel for reduction(+: ntriangle, ncc_triangle, nCrossTaskTriangles) num_threads(li->ntask) schedule(static) for (th = 0; th < li->ntask; th++) { - set_lincs_matrix_task(li, &li->task[th], invmass, - &ncc_triangle, &nCrossTaskTriangles); - ntriangle = li->task[th].ntriangle; + try + { - set_lincs_matrix_task(li, &li->task[th], invmass, &ncc_triangle); ++ set_lincs_matrix_task(li, &li->task[th], invmass, ++ &ncc_triangle, &nCrossTaskTriangles); + ntriangle = li->task[th].ntriangle; + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; } li->ntriangle = ntriangle; li->ncc_triangle = ncc_triangle; diff --cc src/gromacs/mdlib/constr.cpp index 2e9c9b45ae,7a3a215039..5b850d0f56 --- a/src/gromacs/mdlib/constr.cpp +++ b/src/gromacs/mdlib/constr.cpp @@@ -463,30 -458,28 +463,32 @@@ gmx_bool constrain(FILE *fplog, gmx_boo #pragma omp parallel for num_threads(nth) schedule(static) for (th = 0; th < nth; th++) { - int start_th, end_th; - - if (th > 0) + try { - clear_mat(constr->vir_r_m_dr_th[th]); + int start_th, end_th; - constr->settle_error[th] = -1; - } + if (th > 0) + { + clear_mat(constr->vir_r_m_dr_th[th]); + - start_th = (nsettle* th )/nth; - end_th = (nsettle*(th+1))/nth; - if (start_th >= 0 && end_th - start_th > 0) - { - csettle(constr->settled, - end_th-start_th, - settle->iatoms+start_th*(1+NRAL(F_SETTLE)), - pbc_null, - x[0], xprime[0], - invdt, v ? v[0] : NULL, calcvir_atom_end, - th == 0 ? vir_r_m_dr : constr->vir_r_m_dr_th[th], - th == 0 ? &settle_error : &constr->settle_error[th]); ++ constr->settle_error[th] = -1; + } + + start_th = (nsettle* th )/nth; + end_th = (nsettle*(th+1))/nth; + if (start_th >= 0 && end_th - start_th > 0) + { + csettle(constr->settled, + end_th-start_th, + settle->iatoms+start_th*(1+NRAL(F_SETTLE)), + pbc_null, + x[0], xprime[0], + invdt, v ? v[0] : NULL, calcvir_atom_end, + th == 0 ? vir_r_m_dr : constr->vir_r_m_dr_th[th], + th == 0 ? &settle_error : &constr->settle_error[th]); + } } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; } inc_nrnb(nrnb, eNR_SETTLE, nsettle); if (v != NULL) diff --cc src/gromacs/mdlib/nbnxn_search.cpp index d1656c1fa0,0000000000..54a5b31020 mode 100644,000000..100644 --- a/src/gromacs/mdlib/nbnxn_search.cpp +++ b/src/gromacs/mdlib/nbnxn_search.cpp @@@ -1,4136 -1,0 +1,4139 @@@ +/* + * This file is part of the GROMACS molecular simulation package. + * + * Copyright (c) 2012,2013,2014,2015,2016, by the GROMACS development team, led by + * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, + * and including many others, as listed in the AUTHORS file in the + * top-level source directory and at http://www.gromacs.org. + * + * GROMACS is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * GROMACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with GROMACS; if not, see + * http://www.gnu.org/licenses, or write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * If you want to redistribute modifications to GROMACS, please + * consider that scientific software is very special. Version + * control is crucial - bugs must be traceable. We will be happy to + * consider code for inclusion in the official distribution, but + * derived work must not be called official GROMACS. Details are found + * in the README & COPYING files - if they are missing, get the + * official version at http://www.gromacs.org. + * + * To help us fund GROMACS development, we humbly ask that you cite + * the research papers on the package. Check out http://www.gromacs.org. + */ + +#include "gmxpre.h" + +#include "nbnxn_search.h" + +#include "config.h" + +#include +#include + +#include + +#include + +#include "gromacs/domdec/domdec_struct.h" +#include "gromacs/gmxlib/nrnb.h" +#include "gromacs/math/functions.h" +#include "gromacs/math/utilities.h" +#include "gromacs/math/vec.h" +#include "gromacs/mdlib/gmx_omp_nthreads.h" +#include "gromacs/mdlib/nb_verlet.h" +#include "gromacs/mdlib/nbnxn_atomdata.h" +#include "gromacs/mdlib/nbnxn_consts.h" +#include "gromacs/mdlib/nbnxn_grid.h" +#include "gromacs/mdlib/nbnxn_internal.h" +#include "gromacs/mdlib/nbnxn_simd.h" +#include "gromacs/mdlib/nbnxn_util.h" +#include "gromacs/mdlib/ns.h" +#include "gromacs/mdtypes/group.h" +#include "gromacs/mdtypes/md_enums.h" +#include "gromacs/pbcutil/ishift.h" +#include "gromacs/pbcutil/pbc.h" +#include "gromacs/simd/simd.h" +#include "gromacs/simd/vector_operations.h" +#include "gromacs/utility/exceptions.h" +#include "gromacs/utility/fatalerror.h" +#include "gromacs/utility/smalloc.h" + +using namespace gmx; // TODO: Remove when this file is moved into gmx namespace + +#if GMX_SIMD + +/* The functions below are macros as they are performance sensitive */ + +/* 4x4 list, pack=4: no complex conversion required */ +/* i-cluster to j-cluster conversion */ +#define CI_TO_CJ_J4(ci) (ci) +/* cluster index to coordinate array index conversion */ +#define X_IND_CI_J4(ci) ((ci)*STRIDE_P4) +#define X_IND_CJ_J4(cj) ((cj)*STRIDE_P4) + +/* 4x2 list, pack=4: j-cluster size is half the packing width */ +/* i-cluster to j-cluster conversion */ +#define CI_TO_CJ_J2(ci) ((ci)<<1) +/* cluster index to coordinate array index conversion */ +#define X_IND_CI_J2(ci) ((ci)*STRIDE_P4) +#define X_IND_CJ_J2(cj) (((cj)>>1)*STRIDE_P4 + ((cj) & 1)*(PACK_X4>>1)) + +/* 4x8 list, pack=8: i-cluster size is half the packing width */ +/* i-cluster to j-cluster conversion */ +#define CI_TO_CJ_J8(ci) ((ci)>>1) +/* cluster index to coordinate array index conversion */ +#define X_IND_CI_J8(ci) (((ci)>>1)*STRIDE_P8 + ((ci) & 1)*(PACK_X8>>1)) +#define X_IND_CJ_J8(cj) ((cj)*STRIDE_P8) + +/* The j-cluster size is matched to the SIMD width */ +#if GMX_SIMD_REAL_WIDTH == 2 +#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J2(ci) +#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J2(ci) +#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J2(cj) +#else +#if GMX_SIMD_REAL_WIDTH == 4 +#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J4(ci) +#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J4(ci) +#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J4(cj) +#else +#if GMX_SIMD_REAL_WIDTH == 8 +#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J8(ci) +#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J8(ci) +#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J8(cj) +/* Half SIMD with j-cluster size */ +#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J4(ci) +#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J4(ci) +#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J4(cj) +#else +#if GMX_SIMD_REAL_WIDTH == 16 +#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J8(ci) +#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J8(ci) +#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J8(cj) +#else +#error "unsupported GMX_SIMD_REAL_WIDTH" +#endif +#endif +#endif +#endif + +#endif // GMX_SIMD + + +/* We shift the i-particles backward for PBC. + * This leads to more conditionals than shifting forward. + * We do this to get more balanced pair lists. + */ +#define NBNXN_SHIFT_BACKWARD + + +static void nbs_cycle_clear(nbnxn_cycle_t *cc) +{ + for (int i = 0; i < enbsCCnr; i++) + { + cc[i].count = 0; + cc[i].c = 0; + } +} + +static double Mcyc_av(const nbnxn_cycle_t *cc) +{ + return (double)cc->c*1e-6/cc->count; +} + +static void nbs_cycle_print(FILE *fp, const nbnxn_search_t nbs) +{ + fprintf(fp, "\n"); + fprintf(fp, "ns %4d grid %4.1f search %4.1f red.f %5.3f", + nbs->cc[enbsCCgrid].count, + Mcyc_av(&nbs->cc[enbsCCgrid]), + Mcyc_av(&nbs->cc[enbsCCsearch]), + Mcyc_av(&nbs->cc[enbsCCreducef])); + + if (nbs->nthread_max > 1) + { + if (nbs->cc[enbsCCcombine].count > 0) + { + fprintf(fp, " comb %5.2f", + Mcyc_av(&nbs->cc[enbsCCcombine])); + } + fprintf(fp, " s. th"); + for (int t = 0; t < nbs->nthread_max; t++) + { + fprintf(fp, " %4.1f", + Mcyc_av(&nbs->work[t].cc[enbsCCsearch])); + } + } + fprintf(fp, "\n"); +} + +static gmx_inline int ci_to_cj(int na_cj_2log, int ci) +{ + switch (na_cj_2log) + { + case 2: return ci; break; + case 1: return (ci<<1); break; + case 3: return (ci>>1); break; + } + + return 0; +} + +gmx_bool nbnxn_kernel_pairlist_simple(int nb_kernel_type) +{ + if (nb_kernel_type == nbnxnkNotSet) + { + gmx_fatal(FARGS, "Non-bonded kernel type not set for Verlet-style pair-list."); + } + + switch (nb_kernel_type) + { + case nbnxnk8x8x8_GPU: + case nbnxnk8x8x8_PlainC: + return FALSE; + + case nbnxnk4x4_PlainC: + case nbnxnk4xN_SIMD_4xN: + case nbnxnk4xN_SIMD_2xNN: + return TRUE; + + default: + gmx_incons("Invalid nonbonded kernel type passed!"); + return FALSE; + } +} + +/* Initializes a single nbnxn_pairlist_t data structure */ +static void nbnxn_init_pairlist_fep(t_nblist *nl) +{ + nl->type = GMX_NBLIST_INTERACTION_FREE_ENERGY; + nl->igeometry = GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE; + /* The interaction functions are set in the free energy kernel fuction */ + nl->ivdw = -1; + nl->ivdwmod = -1; + nl->ielec = -1; + nl->ielecmod = -1; + + nl->maxnri = 0; + nl->maxnrj = 0; + nl->nri = 0; + nl->nrj = 0; + nl->iinr = NULL; + nl->gid = NULL; + nl->shift = NULL; + nl->jindex = NULL; + nl->jjnr = NULL; + nl->excl_fep = NULL; + +} + +void nbnxn_init_search(nbnxn_search_t * nbs_ptr, + ivec *n_dd_cells, + struct gmx_domdec_zones_t *zones, + gmx_bool bFEP, + int nthread_max) +{ + nbnxn_search_t nbs; + int ngrid; + + snew(nbs, 1); + *nbs_ptr = nbs; + + nbs->bFEP = bFEP; + + nbs->DomDec = (n_dd_cells != NULL); + + clear_ivec(nbs->dd_dim); + ngrid = 1; + if (nbs->DomDec) + { + nbs->zones = zones; + + for (int d = 0; d < DIM; d++) + { + if ((*n_dd_cells)[d] > 1) + { + nbs->dd_dim[d] = 1; + /* Each grid matches a DD zone */ + ngrid *= 2; + } + } + } + + nbnxn_grids_init(nbs, ngrid); + + nbs->cell = NULL; + nbs->cell_nalloc = 0; + nbs->a = NULL; + nbs->a_nalloc = 0; + + nbs->nthread_max = nthread_max; + + /* Initialize the work data structures for each thread */ + snew(nbs->work, nbs->nthread_max); + for (int t = 0; t < nbs->nthread_max; t++) + { + nbs->work[t].cxy_na = NULL; + nbs->work[t].cxy_na_nalloc = 0; + nbs->work[t].sort_work = NULL; + nbs->work[t].sort_work_nalloc = 0; + + snew(nbs->work[t].nbl_fep, 1); + nbnxn_init_pairlist_fep(nbs->work[t].nbl_fep); + } + + /* Initialize detailed nbsearch cycle counting */ + nbs->print_cycles = (getenv("GMX_NBNXN_CYCLE") != 0); + nbs->search_count = 0; + nbs_cycle_clear(nbs->cc); + for (int t = 0; t < nbs->nthread_max; t++) + { + nbs_cycle_clear(nbs->work[t].cc); + } +} + +static void init_buffer_flags(nbnxn_buffer_flags_t *flags, + int natoms) +{ + flags->nflag = (natoms + NBNXN_BUFFERFLAG_SIZE - 1)/NBNXN_BUFFERFLAG_SIZE; + if (flags->nflag > flags->flag_nalloc) + { + flags->flag_nalloc = over_alloc_large(flags->nflag); + srenew(flags->flag, flags->flag_nalloc); + } + for (int b = 0; b < flags->nflag; b++) + { + bitmask_clear(&(flags->flag[b])); + } +} + +/* Determines the cell range along one dimension that + * the bounding box b0 - b1 sees. + */ +static void get_cell_range(real b0, real b1, + int nc, real c0, real s, real invs, + real d2, real r2, int *cf, int *cl) +{ + *cf = std::max(static_cast((b0 - c0)*invs), 0); + + while (*cf > 0 && d2 + gmx::square((b0 - c0) - (*cf-1+1)*s) < r2) + { + (*cf)--; + } + + *cl = std::min(static_cast((b1 - c0)*invs), nc-1); + while (*cl < nc-1 && d2 + gmx::square((*cl+1)*s - (b1 - c0)) < r2) + { + (*cl)++; + } +} + +/* Reference code calculating the distance^2 between two bounding boxes */ +static float box_dist2(float bx0, float bx1, float by0, + float by1, float bz0, float bz1, + const nbnxn_bb_t *bb) +{ + float d2; + float dl, dh, dm, dm0; + + d2 = 0; + + dl = bx0 - bb->upper[BB_X]; + dh = bb->lower[BB_X] - bx1; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + dl = by0 - bb->upper[BB_Y]; + dh = bb->lower[BB_Y] - by1; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + dl = bz0 - bb->upper[BB_Z]; + dh = bb->lower[BB_Z] - bz1; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + return d2; +} + +/* Plain C code calculating the distance^2 between two bounding boxes */ +static float subc_bb_dist2(int si, const nbnxn_bb_t *bb_i_ci, + int csj, const nbnxn_bb_t *bb_j_all) +{ + const nbnxn_bb_t *bb_i, *bb_j; + float d2; + float dl, dh, dm, dm0; + + bb_i = bb_i_ci + si; + bb_j = bb_j_all + csj; + + d2 = 0; + + dl = bb_i->lower[BB_X] - bb_j->upper[BB_X]; + dh = bb_j->lower[BB_X] - bb_i->upper[BB_X]; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + dl = bb_i->lower[BB_Y] - bb_j->upper[BB_Y]; + dh = bb_j->lower[BB_Y] - bb_i->upper[BB_Y]; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + dl = bb_i->lower[BB_Z] - bb_j->upper[BB_Z]; + dh = bb_j->lower[BB_Z] - bb_i->upper[BB_Z]; + dm = std::max(dl, dh); + dm0 = std::max(dm, 0.0f); + d2 += dm0*dm0; + + return d2; +} + +#ifdef NBNXN_SEARCH_BB_SIMD4 + +/* 4-wide SIMD code for bb distance for bb format xyz0 */ +static float subc_bb_dist2_simd4(int si, const nbnxn_bb_t *bb_i_ci, + int csj, const nbnxn_bb_t *bb_j_all) +{ + // TODO: During SIMDv2 transition only some archs use namespace (remove when done) + using namespace gmx; + + Simd4Float bb_i_S0, bb_i_S1; + Simd4Float bb_j_S0, bb_j_S1; + Simd4Float dl_S; + Simd4Float dh_S; + Simd4Float dm_S; + Simd4Float dm0_S; + + bb_i_S0 = load4(&bb_i_ci[si].lower[0]); + bb_i_S1 = load4(&bb_i_ci[si].upper[0]); + bb_j_S0 = load4(&bb_j_all[csj].lower[0]); + bb_j_S1 = load4(&bb_j_all[csj].upper[0]); + + dl_S = bb_i_S0 - bb_j_S1; + dh_S = bb_j_S0 - bb_i_S1; + + dm_S = max(dl_S, dh_S); + dm0_S = max(dm_S, simd4SetZeroF()); + + return dotProduct(dm0_S, dm0_S); +} + +/* Calculate bb bounding distances of bb_i[si,...,si+3] and store them in d2 */ +#define SUBC_BB_DIST2_SIMD4_XXXX_INNER(si, bb_i, d2) \ + { \ + int shi; \ + \ + Simd4Float dx_0, dy_0, dz_0; \ + Simd4Float dx_1, dy_1, dz_1; \ + \ + Simd4Float mx, my, mz; \ + Simd4Float m0x, m0y, m0z; \ + \ + Simd4Float d2x, d2y, d2z; \ + Simd4Float d2s, d2t; \ + \ + shi = si*NNBSBB_D*DIM; \ + \ + xi_l = load4(bb_i+shi+0*STRIDE_PBB); \ + yi_l = load4(bb_i+shi+1*STRIDE_PBB); \ + zi_l = load4(bb_i+shi+2*STRIDE_PBB); \ + xi_h = load4(bb_i+shi+3*STRIDE_PBB); \ + yi_h = load4(bb_i+shi+4*STRIDE_PBB); \ + zi_h = load4(bb_i+shi+5*STRIDE_PBB); \ + \ + dx_0 = xi_l - xj_h; \ + dy_0 = yi_l - yj_h; \ + dz_0 = zi_l - zj_h; \ + \ + dx_1 = xj_l - xi_h; \ + dy_1 = yj_l - yi_h; \ + dz_1 = zj_l - zi_h; \ + \ + mx = max(dx_0, dx_1); \ + my = max(dy_0, dy_1); \ + mz = max(dz_0, dz_1); \ + \ + m0x = max(mx, zero); \ + m0y = max(my, zero); \ + m0z = max(mz, zero); \ + \ + d2x = m0x * m0x; \ + d2y = m0y * m0y; \ + d2z = m0z * m0z; \ + \ + d2s = d2x + d2y; \ + d2t = d2s + d2z; \ + \ + store4(d2+si, d2t); \ + } + +/* 4-wide SIMD code for nsi bb distances for bb format xxxxyyyyzzzz */ +static void subc_bb_dist2_simd4_xxxx(const float *bb_j, + int nsi, const float *bb_i, + float *d2) +{ + // TODO: During SIMDv2 transition only some archs use namespace (remove when done) + using namespace gmx; + + Simd4Float xj_l, yj_l, zj_l; + Simd4Float xj_h, yj_h, zj_h; + Simd4Float xi_l, yi_l, zi_l; + Simd4Float xi_h, yi_h, zi_h; + + Simd4Float zero; + + zero = setZero(); + + xj_l = Simd4Float(bb_j[0*STRIDE_PBB]); + yj_l = Simd4Float(bb_j[1*STRIDE_PBB]); + zj_l = Simd4Float(bb_j[2*STRIDE_PBB]); + xj_h = Simd4Float(bb_j[3*STRIDE_PBB]); + yj_h = Simd4Float(bb_j[4*STRIDE_PBB]); + zj_h = Simd4Float(bb_j[5*STRIDE_PBB]); + + /* Here we "loop" over si (0,STRIDE_PBB) from 0 to nsi with step STRIDE_PBB. + * But as we know the number of iterations is 1 or 2, we unroll manually. + */ + SUBC_BB_DIST2_SIMD4_XXXX_INNER(0, bb_i, d2); + if (STRIDE_PBB < nsi) + { + SUBC_BB_DIST2_SIMD4_XXXX_INNER(STRIDE_PBB, bb_i, d2); + } +} + +#endif /* NBNXN_SEARCH_BB_SIMD4 */ + + +/* Returns if any atom pair from two clusters is within distance sqrt(rl2) */ +static gmx_inline gmx_bool +clusterpair_in_range(const nbnxn_list_work_t *work, + int si, + int csj, int stride, const real *x_j, + real rl2) +{ +#ifndef NBNXN_SEARCH_BB_SIMD4 + + /* Plain C version. + * All coordinates are stored as xyzxyz... + */ + + const real *x_i = work->x_ci; + + for (int i = 0; i < nbnxn_gpu_cluster_size; i++) + { + int i0 = (si*nbnxn_gpu_cluster_size + i)*DIM; + for (int j = 0; j < nbnxn_gpu_cluster_size; j++) + { + int j0 = (csj*nbnxn_gpu_cluster_size + j)*stride; + + real d2 = gmx::square(x_i[i0 ] - x_j[j0 ]) + gmx::square(x_i[i0+1] - x_j[j0+1]) + gmx::square(x_i[i0+2] - x_j[j0+2]); + + if (d2 < rl2) + { + return TRUE; + } + } + } + + return FALSE; + +#else /* !NBNXN_SEARCH_BB_SIMD4 */ + + /* 4-wide SIMD version. + * A cluster is hard-coded to 8 atoms. + * The coordinates x_i are stored as xxxxyyyy..., x_j is stored xyzxyz... + * Using 8-wide AVX is not faster on Intel Sandy Bridge. + */ + assert(nbnxn_gpu_cluster_size == 8); + + Simd4Real rc2_S = Simd4Real(rl2); + + const real *x_i = work->x_ci_simd; + + int dim_stride = nbnxn_gpu_cluster_size*DIM; + Simd4Real ix_S0 = load4(x_i + si*dim_stride + 0*GMX_SIMD4_WIDTH); + Simd4Real iy_S0 = load4(x_i + si*dim_stride + 1*GMX_SIMD4_WIDTH); + Simd4Real iz_S0 = load4(x_i + si*dim_stride + 2*GMX_SIMD4_WIDTH); + Simd4Real ix_S1 = load4(x_i + si*dim_stride + 3*GMX_SIMD4_WIDTH); + Simd4Real iy_S1 = load4(x_i + si*dim_stride + 4*GMX_SIMD4_WIDTH); + Simd4Real iz_S1 = load4(x_i + si*dim_stride + 5*GMX_SIMD4_WIDTH); + + /* We loop from the outer to the inner particles to maximize + * the chance that we find a pair in range quickly and return. + */ + int j0 = csj*nbnxn_gpu_cluster_size; + int j1 = j0 + nbnxn_gpu_cluster_size - 1; + while (j0 < j1) + { + Simd4Real jx0_S, jy0_S, jz0_S; + Simd4Real jx1_S, jy1_S, jz1_S; + + Simd4Real dx_S0, dy_S0, dz_S0; + Simd4Real dx_S1, dy_S1, dz_S1; + Simd4Real dx_S2, dy_S2, dz_S2; + Simd4Real dx_S3, dy_S3, dz_S3; + + Simd4Real rsq_S0; + Simd4Real rsq_S1; + Simd4Real rsq_S2; + Simd4Real rsq_S3; + + Simd4Bool wco_S0; + Simd4Bool wco_S1; + Simd4Bool wco_S2; + Simd4Bool wco_S3; + Simd4Bool wco_any_S01, wco_any_S23, wco_any_S; + + jx0_S = Simd4Real(x_j[j0*stride+0]); + jy0_S = Simd4Real(x_j[j0*stride+1]); + jz0_S = Simd4Real(x_j[j0*stride+2]); + + jx1_S = Simd4Real(x_j[j1*stride+0]); + jy1_S = Simd4Real(x_j[j1*stride+1]); + jz1_S = Simd4Real(x_j[j1*stride+2]); + + /* Calculate distance */ + dx_S0 = ix_S0 - jx0_S; + dy_S0 = iy_S0 - jy0_S; + dz_S0 = iz_S0 - jz0_S; + dx_S1 = ix_S1 - jx0_S; + dy_S1 = iy_S1 - jy0_S; + dz_S1 = iz_S1 - jz0_S; + dx_S2 = ix_S0 - jx1_S; + dy_S2 = iy_S0 - jy1_S; + dz_S2 = iz_S0 - jz1_S; + dx_S3 = ix_S1 - jx1_S; + dy_S3 = iy_S1 - jy1_S; + dz_S3 = iz_S1 - jz1_S; + + /* rsq = dx*dx+dy*dy+dz*dz */ + rsq_S0 = norm2(dx_S0, dy_S0, dz_S0); + rsq_S1 = norm2(dx_S1, dy_S1, dz_S1); + rsq_S2 = norm2(dx_S2, dy_S2, dz_S2); + rsq_S3 = norm2(dx_S3, dy_S3, dz_S3); + + wco_S0 = (rsq_S0 < rc2_S); + wco_S1 = (rsq_S1 < rc2_S); + wco_S2 = (rsq_S2 < rc2_S); + wco_S3 = (rsq_S3 < rc2_S); + + wco_any_S01 = wco_S0 || wco_S1; + wco_any_S23 = wco_S2 || wco_S3; + wco_any_S = wco_any_S01 || wco_any_S23; + + if (anyTrue(wco_any_S)) + { + return TRUE; + } + + j0++; + j1--; + } + + return FALSE; + +#endif /* !NBNXN_SEARCH_BB_SIMD4 */ +} + +/* Returns the j sub-cell for index cj_ind */ +static int nbl_cj(const nbnxn_pairlist_t *nbl, int cj_ind) +{ + return nbl->cj4[cj_ind/nbnxn_gpu_jgroup_size].cj[cj_ind & (nbnxn_gpu_jgroup_size - 1)]; +} + +/* Returns the i-interaction mask of the j sub-cell for index cj_ind */ +static unsigned int nbl_imask0(const nbnxn_pairlist_t *nbl, int cj_ind) +{ + return nbl->cj4[cj_ind/nbnxn_gpu_jgroup_size].imei[0].imask; +} + +/* Ensures there is enough space for extra extra exclusion masks */ +static void check_excl_space(nbnxn_pairlist_t *nbl, int extra) +{ + if (nbl->nexcl+extra > nbl->excl_nalloc) + { + nbl->excl_nalloc = over_alloc_small(nbl->nexcl+extra); + nbnxn_realloc_void((void **)&nbl->excl, + nbl->nexcl*sizeof(*nbl->excl), + nbl->excl_nalloc*sizeof(*nbl->excl), + nbl->alloc, nbl->free); + } +} + +/* Ensures there is enough space for ncell extra j-cells in the list */ +static void check_cell_list_space_simple(nbnxn_pairlist_t *nbl, + int ncell) +{ + int cj_max; + + cj_max = nbl->ncj + ncell; + + if (cj_max > nbl->cj_nalloc) + { + nbl->cj_nalloc = over_alloc_small(cj_max); + nbnxn_realloc_void((void **)&nbl->cj, + nbl->ncj*sizeof(*nbl->cj), + nbl->cj_nalloc*sizeof(*nbl->cj), + nbl->alloc, nbl->free); + } +} + +/* Ensures there is enough space for ncell extra j-clusters in the list */ +static void check_cell_list_space_supersub(nbnxn_pairlist_t *nbl, + int ncell) +{ + int ncj4_max, w; + + /* We can have maximally nsupercell*gpu_ncluster_per_cell sj lists */ + /* We can store 4 j-subcell - i-supercell pairs in one struct. + * since we round down, we need one extra entry. + */ + ncj4_max = ((nbl->work->cj_ind + ncell*gpu_ncluster_per_cell + nbnxn_gpu_jgroup_size - 1)/nbnxn_gpu_jgroup_size); + + if (ncj4_max > nbl->cj4_nalloc) + { + nbl->cj4_nalloc = over_alloc_small(ncj4_max); + nbnxn_realloc_void((void **)&nbl->cj4, + nbl->work->cj4_init*sizeof(*nbl->cj4), + nbl->cj4_nalloc*sizeof(*nbl->cj4), + nbl->alloc, nbl->free); + } + + if (ncj4_max > nbl->work->cj4_init) + { + for (int j4 = nbl->work->cj4_init; j4 < ncj4_max; j4++) + { + /* No i-subcells and no excl's in the list initially */ + for (w = 0; w < nbnxn_gpu_clusterpair_split; w++) + { + nbl->cj4[j4].imei[w].imask = 0U; + nbl->cj4[j4].imei[w].excl_ind = 0; + + } + } + nbl->work->cj4_init = ncj4_max; + } +} + +/* Set all excl masks for one GPU warp no exclusions */ +static void set_no_excls(nbnxn_excl_t *excl) +{ + for (int t = 0; t < nbnxn_gpu_excl_size; t++) + { + /* Turn all interaction bits on */ + excl->pair[t] = NBNXN_INTERACTION_MASK_ALL; + } +} + +/* Initializes a single nbnxn_pairlist_t data structure */ +static void nbnxn_init_pairlist(nbnxn_pairlist_t *nbl, + gmx_bool bSimple, + nbnxn_alloc_t *alloc, + nbnxn_free_t *free) +{ + if (alloc == NULL) + { + nbl->alloc = nbnxn_alloc_aligned; + } + else + { + nbl->alloc = alloc; + } + if (free == NULL) + { + nbl->free = nbnxn_free_aligned; + } + else + { + nbl->free = free; + } + + nbl->bSimple = bSimple; + nbl->na_sc = 0; + nbl->na_ci = 0; + nbl->na_cj = 0; + nbl->nci = 0; + nbl->ci = NULL; + nbl->ci_nalloc = 0; ++ nbl->nsci = 0; ++ nbl->sci = NULL; ++ nbl->sci_nalloc = 0; + nbl->ncj = 0; + nbl->cj = NULL; + nbl->cj_nalloc = 0; + nbl->ncj4 = 0; + /* We need one element extra in sj, so alloc initially with 1 */ + nbl->cj4_nalloc = 0; + nbl->cj4 = NULL; + nbl->nci_tot = 0; + + if (!nbl->bSimple) + { + GMX_ASSERT(nbnxn_gpu_ncluster_per_supercluster == gpu_ncluster_per_cell, "The search code assumes that the a super-cluster matches a search grid cell"); + + GMX_ASSERT(sizeof(nbl->cj4[0].imei[0].imask)*8 >= nbnxn_gpu_jgroup_size*gpu_ncluster_per_cell, "The i super-cluster cluster interaction mask does not contain a sufficient number of bits"); + GMX_ASSERT(sizeof(nbl->excl[0])*8 >= nbnxn_gpu_jgroup_size*gpu_ncluster_per_cell, "The GPU exclusion mask does not contain a sufficient number of bits"); + + nbl->excl = NULL; + nbl->excl_nalloc = 0; + nbl->nexcl = 0; + check_excl_space(nbl, 1); + nbl->nexcl = 1; + set_no_excls(&nbl->excl[0]); + } + + snew(nbl->work, 1); + if (nbl->bSimple) + { + snew_aligned(nbl->work->bb_ci, 1, NBNXN_SEARCH_BB_MEM_ALIGN); + } + else + { +#ifdef NBNXN_BBXXXX + snew_aligned(nbl->work->pbb_ci, gpu_ncluster_per_cell/STRIDE_PBB*NNBSBB_XXXX, NBNXN_SEARCH_BB_MEM_ALIGN); +#else + snew_aligned(nbl->work->bb_ci, gpu_ncluster_per_cell, NBNXN_SEARCH_BB_MEM_ALIGN); +#endif + } + int gpu_clusterpair_nc = gpu_ncluster_per_cell*nbnxn_gpu_cluster_size*DIM; + snew(nbl->work->x_ci, gpu_clusterpair_nc); +#if GMX_SIMD + snew_aligned(nbl->work->x_ci_simd, + std::max(NBNXN_CPU_CLUSTER_I_SIZE*DIM*GMX_SIMD_REAL_WIDTH, + gpu_clusterpair_nc), + GMX_SIMD_REAL_WIDTH); +#endif + snew_aligned(nbl->work->d2, gpu_ncluster_per_cell, NBNXN_SEARCH_BB_MEM_ALIGN); + + nbl->work->sort = NULL; + nbl->work->sort_nalloc = 0; + nbl->work->sci_sort = NULL; + nbl->work->sci_sort_nalloc = 0; +} + +void nbnxn_init_pairlist_set(nbnxn_pairlist_set_t *nbl_list, + gmx_bool bSimple, gmx_bool bCombined, + nbnxn_alloc_t *alloc, + nbnxn_free_t *free) +{ + nbl_list->bSimple = bSimple; + nbl_list->bCombined = bCombined; + + nbl_list->nnbl = gmx_omp_nthreads_get(emntNonbonded); + + if (!nbl_list->bCombined && + nbl_list->nnbl > NBNXN_BUFFERFLAG_MAX_THREADS) + { + gmx_fatal(FARGS, "%d OpenMP threads were requested. Since the non-bonded force buffer reduction is prohibitively slow with more than %d threads, we do not allow this. Use %d or less OpenMP threads.", + nbl_list->nnbl, NBNXN_BUFFERFLAG_MAX_THREADS, NBNXN_BUFFERFLAG_MAX_THREADS); + } + + snew(nbl_list->nbl, nbl_list->nnbl); + snew(nbl_list->nbl_fep, nbl_list->nnbl); + /* Execute in order to avoid memory interleaving between threads */ +#pragma omp parallel for num_threads(nbl_list->nnbl) schedule(static) + for (int i = 0; i < nbl_list->nnbl; i++) + { + try + { + /* Allocate the nblist data structure locally on each thread + * to optimize memory access for NUMA architectures. + */ + snew(nbl_list->nbl[i], 1); + + /* Only list 0 is used on the GPU, use normal allocation for i>0 */ + if (i == 0) + { + nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, alloc, free); + } + else + { + nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, NULL, NULL); + } + + snew(nbl_list->nbl_fep[i], 1); + nbnxn_init_pairlist_fep(nbl_list->nbl_fep[i]); + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + } +} + +/* Print statistics of a pair list, used for debug output */ +static void print_nblist_statistics_simple(FILE *fp, const nbnxn_pairlist_t *nbl, + const nbnxn_search_t nbs, real rl) +{ + const nbnxn_grid_t *grid; + int cs[SHIFTS]; + int npexcl; + + /* This code only produces correct statistics with domain decomposition */ + grid = &nbs->grid[0]; + + fprintf(fp, "nbl nci %d ncj %d\n", + nbl->nci, nbl->ncj); + fprintf(fp, "nbl na_sc %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n", + nbl->na_sc, rl, nbl->ncj, nbl->ncj/(double)grid->nc, + nbl->ncj/(double)grid->nc*grid->na_sc, + nbl->ncj/(double)grid->nc*grid->na_sc/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nc*grid->na_sc/(grid->size[XX]*grid->size[YY]*grid->size[ZZ]))); + + fprintf(fp, "nbl average j cell list length %.1f\n", + 0.25*nbl->ncj/(double)std::max(nbl->nci, 1)); + + for (int s = 0; s < SHIFTS; s++) + { + cs[s] = 0; + } + npexcl = 0; + for (int i = 0; i < nbl->nci; i++) + { + cs[nbl->ci[i].shift & NBNXN_CI_SHIFT] += + nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start; + + int j = nbl->ci[i].cj_ind_start; + while (j < nbl->ci[i].cj_ind_end && + nbl->cj[j].excl != NBNXN_INTERACTION_MASK_ALL) + { + npexcl++; + j++; + } + } + fprintf(fp, "nbl cell pairs, total: %d excl: %d %.1f%%\n", + nbl->ncj, npexcl, 100*npexcl/(double)std::max(nbl->ncj, 1)); + for (int s = 0; s < SHIFTS; s++) + { + if (cs[s] > 0) + { + fprintf(fp, "nbl shift %2d ncj %3d\n", s, cs[s]); + } + } +} + +/* Print statistics of a pair lists, used for debug output */ +static void print_nblist_statistics_supersub(FILE *fp, const nbnxn_pairlist_t *nbl, + const nbnxn_search_t nbs, real rl) +{ + const nbnxn_grid_t *grid; + int b; + int c[gpu_ncluster_per_cell + 1]; + double sum_nsp, sum_nsp2; + int nsp_max; + + /* This code only produces correct statistics with domain decomposition */ + grid = &nbs->grid[0]; + + fprintf(fp, "nbl nsci %d ncj4 %d nsi %d excl4 %d\n", + nbl->nsci, nbl->ncj4, nbl->nci_tot, nbl->nexcl); + fprintf(fp, "nbl na_c %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n", + nbl->na_ci, rl, nbl->nci_tot, nbl->nci_tot/(double)grid->nsubc_tot, + nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c, + nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nsubc_tot*grid->na_c/(grid->size[XX]*grid->size[YY]*grid->size[ZZ]))); + + sum_nsp = 0; + sum_nsp2 = 0; + nsp_max = 0; + for (int si = 0; si <= gpu_ncluster_per_cell; si++) + { + c[si] = 0; + } + for (int i = 0; i < nbl->nsci; i++) + { + int nsp; + + nsp = 0; + for (int j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++) + { + for (int j = 0; j < nbnxn_gpu_jgroup_size; j++) + { + b = 0; + for (int si = 0; si < gpu_ncluster_per_cell; si++) + { + if (nbl->cj4[j4].imei[0].imask & (1U << (j*gpu_ncluster_per_cell + si))) + { + b++; + } + } + nsp += b; + c[b]++; + } + } + sum_nsp += nsp; + sum_nsp2 += nsp*nsp; + nsp_max = std::max(nsp_max, nsp); + } + if (nbl->nsci > 0) + { + sum_nsp /= nbl->nsci; + sum_nsp2 /= nbl->nsci; + } + fprintf(fp, "nbl #cluster-pairs: av %.1f stddev %.1f max %d\n", + sum_nsp, std::sqrt(sum_nsp2 - sum_nsp*sum_nsp), nsp_max); + + if (nbl->ncj4 > 0) + { + for (b = 0; b <= gpu_ncluster_per_cell; b++) + { + fprintf(fp, "nbl j-list #i-subcell %d %7d %4.1f\n", + b, c[b], + 100.0*c[b]/(double)(nbl->ncj4*nbnxn_gpu_jgroup_size)); + } + } +} + +/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp */ +static void low_get_nbl_exclusions(nbnxn_pairlist_t *nbl, int cj4, + int warp, nbnxn_excl_t **excl) +{ + if (nbl->cj4[cj4].imei[warp].excl_ind == 0) + { + /* No exclusions set, make a new list entry */ + nbl->cj4[cj4].imei[warp].excl_ind = nbl->nexcl; + nbl->nexcl++; + *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind]; + set_no_excls(*excl); + } + else + { + /* We already have some exclusions, new ones can be added to the list */ + *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind]; + } +} + +/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp, + * generates a new element and allocates extra memory, if necessary. + */ +static void get_nbl_exclusions_1(nbnxn_pairlist_t *nbl, int cj4, + int warp, nbnxn_excl_t **excl) +{ + if (nbl->cj4[cj4].imei[warp].excl_ind == 0) + { + /* We need to make a new list entry, check if we have space */ + check_excl_space(nbl, 1); + } + low_get_nbl_exclusions(nbl, cj4, warp, excl); +} + +/* Returns pointers to the exclusion masks for cj4-unit cj4 for both warps, + * generates a new element and allocates extra memory, if necessary. + */ +static void get_nbl_exclusions_2(nbnxn_pairlist_t *nbl, int cj4, + nbnxn_excl_t **excl_w0, + nbnxn_excl_t **excl_w1) +{ + /* Check for space we might need */ + check_excl_space(nbl, 2); + + low_get_nbl_exclusions(nbl, cj4, 0, excl_w0); + low_get_nbl_exclusions(nbl, cj4, 1, excl_w1); +} + +/* Sets the self exclusions i=j and pair exclusions i>j */ +static void set_self_and_newton_excls_supersub(nbnxn_pairlist_t *nbl, + int cj4_ind, int sj_offset, + int i_cluster_in_cell) +{ + nbnxn_excl_t *excl[nbnxn_gpu_clusterpair_split]; + + /* Here we only set the set self and double pair exclusions */ + + assert(nbnxn_gpu_clusterpair_split == 2); + + get_nbl_exclusions_2(nbl, cj4_ind, &excl[0], &excl[1]); + + /* Only minor < major bits set */ + for (int ej = 0; ej < nbl->na_ci; ej++) + { + int w = (ej>>2); + for (int ei = ej; ei < nbl->na_ci; ei++) + { + excl[w]->pair[(ej & (nbnxn_gpu_jgroup_size-1))*nbl->na_ci + ei] &= + ~(1U << (sj_offset*gpu_ncluster_per_cell + i_cluster_in_cell)); + } + } +} + +/* Returns a diagonal or off-diagonal interaction mask for plain C lists */ +static unsigned int get_imask(gmx_bool rdiag, int ci, int cj) +{ + return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL); +} + +/* Returns a diagonal or off-diagonal interaction mask for cj-size=2 */ +static unsigned int get_imask_simd_j2(gmx_bool rdiag, int ci, int cj) +{ + return (rdiag && ci*2 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_0 : + (rdiag && ci*2+1 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_1 : + NBNXN_INTERACTION_MASK_ALL)); +} + +/* Returns a diagonal or off-diagonal interaction mask for cj-size=4 */ +static unsigned int get_imask_simd_j4(gmx_bool rdiag, int ci, int cj) +{ + return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL); +} + +/* Returns a diagonal or off-diagonal interaction mask for cj-size=8 */ +static unsigned int get_imask_simd_j8(gmx_bool rdiag, int ci, int cj) +{ + return (rdiag && ci == cj*2 ? NBNXN_INTERACTION_MASK_DIAG_J8_0 : + (rdiag && ci == cj*2+1 ? NBNXN_INTERACTION_MASK_DIAG_J8_1 : + NBNXN_INTERACTION_MASK_ALL)); +} + +#if GMX_SIMD +#if GMX_SIMD_REAL_WIDTH == 2 +#define get_imask_simd_4xn get_imask_simd_j2 +#endif +#if GMX_SIMD_REAL_WIDTH == 4 +#define get_imask_simd_4xn get_imask_simd_j4 +#endif +#if GMX_SIMD_REAL_WIDTH == 8 +#define get_imask_simd_4xn get_imask_simd_j8 +#define get_imask_simd_2xnn get_imask_simd_j4 +#endif +#if GMX_SIMD_REAL_WIDTH == 16 +#define get_imask_simd_2xnn get_imask_simd_j8 +#endif +#endif + +/* Plain C code for making a pair list of cell ci vs cell cjf-cjl. + * Checks bounding box distances and possibly atom pair distances. + */ +static void make_cluster_list_simple(const nbnxn_grid_t *gridj, + nbnxn_pairlist_t *nbl, + int ci, int cjf, int cjl, + gmx_bool remove_sub_diag, + const real *x_j, + real rl2, float rbb2, + int *ndistc) +{ + const nbnxn_bb_t *bb_ci; + const real *x_ci; + + gmx_bool InRange; + real d2; + int cjf_gl, cjl_gl; + + bb_ci = nbl->work->bb_ci; + x_ci = nbl->work->x_ci; + + InRange = FALSE; + while (!InRange && cjf <= cjl) + { + d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bb); + *ndistc += 2; + + /* Check if the distance is within the distance where + * we use only the bounding box distance rbb, + * or within the cut-off and there is at least one atom pair + * within the cut-off. + */ + if (d2 < rbb2) + { + InRange = TRUE; + } + else if (d2 < rl2) + { + cjf_gl = gridj->cell0 + cjf; + for (int i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++) + { + for (int j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++) + { + InRange = InRange || + (gmx::square(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) + + gmx::square(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) + + gmx::square(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2); + } + } + *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE; + } + if (!InRange) + { + cjf++; + } + } + if (!InRange) + { + return; + } + + InRange = FALSE; + while (!InRange && cjl > cjf) + { + d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bb); + *ndistc += 2; + + /* Check if the distance is within the distance where + * we use only the bounding box distance rbb, + * or within the cut-off and there is at least one atom pair + * within the cut-off. + */ + if (d2 < rbb2) + { + InRange = TRUE; + } + else if (d2 < rl2) + { + cjl_gl = gridj->cell0 + cjl; + for (int i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++) + { + for (int j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++) + { + InRange = InRange || + (gmx::square(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) + + gmx::square(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) + + gmx::square(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2); + } + } + *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE; + } + if (!InRange) + { + cjl--; + } + } + + if (cjf <= cjl) + { + for (int cj = cjf; cj <= cjl; cj++) + { + /* Store cj and the interaction mask */ + nbl->cj[nbl->ncj].cj = gridj->cell0 + cj; + nbl->cj[nbl->ncj].excl = get_imask(remove_sub_diag, ci, cj); + nbl->ncj++; + } + /* Increase the closing index in i super-cell list */ + nbl->ci[nbl->nci].cj_ind_end = nbl->ncj; + } +} + +#ifdef GMX_NBNXN_SIMD_4XN +#include "gromacs/mdlib/nbnxn_search_simd_4xn.h" +#endif +#ifdef GMX_NBNXN_SIMD_2XNN +#include "gromacs/mdlib/nbnxn_search_simd_2xnn.h" +#endif + +/* Plain C or SIMD4 code for making a pair list of super-cell sci vs scj. + * Checks bounding box distances and possibly atom pair distances. + */ +static void make_cluster_list_supersub(const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj, + nbnxn_pairlist_t *nbl, + int sci, int scj, + gmx_bool sci_equals_scj, + int stride, const real *x, + real rl2, float rbb2, + int *ndistc) +{ + nbnxn_list_work_t *work = nbl->work; + +#ifdef NBNXN_BBXXXX + const float *pbb_ci = work->pbb_ci; +#else + const nbnxn_bb_t *bb_ci = work->bb_ci; +#endif + + const int na_c = nbnxn_gpu_cluster_size; + assert(na_c == gridi->na_c); + assert(na_c == gridj->na_c); + + /* We generate the pairlist mainly based on bounding-box distances + * and do atom pair distance based pruning on the GPU. + * Only if a j-group contains a single cluster-pair, we try to prune + * that pair based on atom distances on the CPU to avoid empty j-groups. + */ +#define PRUNE_LIST_CPU_ONE +#ifdef PRUNE_LIST_CPU_ONE + int ci_last = -1; +#endif + + float *d2l = work->d2; + + for (int subc = 0; subc < gridj->nsubc[scj]; subc++) + { + int cj4_ind = nbl->work->cj_ind/nbnxn_gpu_jgroup_size; + int cj_offset = nbl->work->cj_ind - cj4_ind*nbnxn_gpu_jgroup_size; + nbnxn_cj4_t *cj4 = &nbl->cj4[cj4_ind]; + + int cj = scj*gpu_ncluster_per_cell + subc; + + int cj_gl = gridj->cell0*gpu_ncluster_per_cell + cj; + + /* Initialize this j-subcell i-subcell list */ + cj4->cj[cj_offset] = cj_gl; + + int ci1; + if (sci_equals_scj) + { + ci1 = subc + 1; + } + else + { + ci1 = gridi->nsubc[sci]; + } + +#ifdef NBNXN_BBXXXX + /* Determine all ci1 bb distances in one call with SIMD4 */ + subc_bb_dist2_simd4_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)), + ci1, pbb_ci, d2l); + *ndistc += na_c*2; +#endif + + int npair = 0; + unsigned int imask = 0; + /* We use a fixed upper-bound instead of ci1 to help optimization */ + for (int ci = 0; ci < gpu_ncluster_per_cell; ci++) + { + if (ci == ci1) + { + break; + } + +#ifndef NBNXN_BBXXXX + /* Determine the bb distance between ci and cj */ + d2l[ci] = subc_bb_dist2(ci, bb_ci, cj, gridj->bb); + *ndistc += 2; +#endif + float d2 = d2l[ci]; + +#ifdef PRUNE_LIST_CPU_ALL + /* Check if the distance is within the distance where + * we use only the bounding box distance rbb, + * or within the cut-off and there is at least one atom pair + * within the cut-off. This check is very costly. + */ + *ndistc += na_c*na_c; + if (d2 < rbb2 || + (d2 < rl2 && + clusterpair_in_range(work, ci, cj_gl, stride, x, rl2))) +#else + /* Check if the distance between the two bounding boxes + * in within the pair-list cut-off. + */ + if (d2 < rl2) +#endif + { + /* Flag this i-subcell to be taken into account */ + imask |= (1U << (cj_offset*gpu_ncluster_per_cell + ci)); + +#ifdef PRUNE_LIST_CPU_ONE + ci_last = ci; +#endif + + npair++; + } + } + +#ifdef PRUNE_LIST_CPU_ONE + /* If we only found 1 pair, check if any atoms are actually + * within the cut-off, so we could get rid of it. + */ + if (npair == 1 && d2l[ci_last] >= rbb2 && + !clusterpair_in_range(work, ci_last, cj_gl, stride, x, rl2)) + { + imask &= ~(1U << (cj_offset*gpu_ncluster_per_cell + ci_last)); + npair--; + } +#endif + + if (npair > 0) + { + /* We have a useful sj entry, close it now */ + + /* Set the exclucions for the ci== sj entry. + * Here we don't bother to check if this entry is actually flagged, + * as it will nearly always be in the list. + */ + if (sci_equals_scj) + { + set_self_and_newton_excls_supersub(nbl, cj4_ind, cj_offset, subc); + } + + /* Copy the cluster interaction mask to the list */ + for (int w = 0; w < nbnxn_gpu_clusterpair_split; w++) + { + cj4->imei[w].imask |= imask; + } + + nbl->work->cj_ind++; + + /* Keep the count */ + nbl->nci_tot += npair; + + /* Increase the closing index in i super-cell list */ + nbl->sci[nbl->nsci].cj4_ind_end = + (nbl->work->cj_ind + nbnxn_gpu_jgroup_size - 1)/nbnxn_gpu_jgroup_size; + } + } +} + +/* Set all atom-pair exclusions from the topology stored in excl + * as masks in the pair-list for simple list i-entry nbl_ci + */ +static void set_ci_top_excls(const nbnxn_search_t nbs, + nbnxn_pairlist_t *nbl, + gmx_bool diagRemoved, + int na_ci_2log, + int na_cj_2log, + const nbnxn_ci_t *nbl_ci, + const t_blocka *excl) +{ + const int *cell; + int ci; + int cj_ind_first, cj_ind_last; + int cj_first, cj_last; + int ndirect; + int ai, aj, si, ge, se; + int found, cj_ind_0, cj_ind_1, cj_ind_m; + int cj_m; + int inner_i, inner_e; + + cell = nbs->cell; + + if (nbl_ci->cj_ind_end == nbl_ci->cj_ind_start) + { + /* Empty list */ + return; + } + + ci = nbl_ci->ci; + + cj_ind_first = nbl_ci->cj_ind_start; + cj_ind_last = nbl->ncj - 1; + + cj_first = nbl->cj[cj_ind_first].cj; + cj_last = nbl->cj[cj_ind_last].cj; + + /* Determine how many contiguous j-cells we have starting + * from the first i-cell. This number can be used to directly + * calculate j-cell indices for excluded atoms. + */ + ndirect = 0; + if (na_ci_2log == na_cj_2log) + { + while (cj_ind_first + ndirect <= cj_ind_last && + nbl->cj[cj_ind_first+ndirect].cj == ci + ndirect) + { + ndirect++; + } + } +#ifdef NBNXN_SEARCH_BB_SIMD4 + else + { + while (cj_ind_first + ndirect <= cj_ind_last && + nbl->cj[cj_ind_first+ndirect].cj == ci_to_cj(na_cj_2log, ci) + ndirect) + { + ndirect++; + } + } +#endif + + /* Loop over the atoms in the i super-cell */ + for (int i = 0; i < nbl->na_sc; i++) + { + ai = nbs->a[ci*nbl->na_sc+i]; + if (ai >= 0) + { + si = (i>>na_ci_2log); + + /* Loop over the topology-based exclusions for this i-atom */ + for (int eind = excl->index[ai]; eind < excl->index[ai+1]; eind++) + { + aj = excl->a[eind]; + + if (aj == ai) + { + /* The self exclusion are already set, save some time */ + continue; + } + + ge = cell[aj]; + + /* Without shifts we only calculate interactions j>i + * for one-way pair-lists. + */ + if (diagRemoved && ge <= ci*nbl->na_sc + i) + { + continue; + } + + se = (ge >> na_cj_2log); + + /* Could the cluster se be in our list? */ + if (se >= cj_first && se <= cj_last) + { + if (se < cj_first + ndirect) + { + /* We can calculate cj_ind directly from se */ + found = cj_ind_first + se - cj_first; + } + else + { + /* Search for se using bisection */ + found = -1; + cj_ind_0 = cj_ind_first + ndirect; + cj_ind_1 = cj_ind_last + 1; + while (found == -1 && cj_ind_0 < cj_ind_1) + { + cj_ind_m = (cj_ind_0 + cj_ind_1)>>1; + + cj_m = nbl->cj[cj_ind_m].cj; + + if (se == cj_m) + { + found = cj_ind_m; + } + else if (se < cj_m) + { + cj_ind_1 = cj_ind_m; + } + else + { + cj_ind_0 = cj_ind_m + 1; + } + } + } + + if (found >= 0) + { + inner_i = i - (si << na_ci_2log); + inner_e = ge - (se << na_cj_2log); + + nbl->cj[found].excl &= ~(1U<<((inner_i<nri++; + + assert(nlist->nri < nlist->maxnri); + + /* Duplicate the last i-entry, except for jindex, which continues */ + nlist->iinr[nlist->nri] = nlist->iinr[nlist->nri-1]; + nlist->shift[nlist->nri] = nlist->shift[nlist->nri-1]; + nlist->gid[nlist->nri] = nlist->gid[nlist->nri-1]; + nlist->jindex[nlist->nri] = nlist->nrj; +} + +/* For load balancing of the free-energy lists over threads, we set + * the maximum nrj size of an i-entry to 40. This leads to good + * load balancing in the worst case scenario of a single perturbed + * particle on 16 threads, while not introducing significant overhead. + * Note that half of the perturbed pairs will anyhow end up in very small lists, + * since non perturbed i-particles will see few perturbed j-particles). + */ +const int max_nrj_fep = 40; + +/* Exclude the perturbed pairs from the Verlet list. This is only done to avoid + * singularities for overlapping particles (0/0), since the charges and + * LJ parameters have been zeroed in the nbnxn data structure. + * Simultaneously make a group pair list for the perturbed pairs. + */ +static void make_fep_list(const nbnxn_search_t nbs, + const nbnxn_atomdata_t *nbat, + nbnxn_pairlist_t *nbl, + gmx_bool bDiagRemoved, + nbnxn_ci_t *nbl_ci, + const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj, + t_nblist *nlist) +{ + int ci, cj_ind_start, cj_ind_end, cja, cjr; + int nri_max; + int ngid, gid_i = 0, gid_j, gid; + int egp_shift, egp_mask; + int gid_cj = 0; + int ind_i, ind_j, ai, aj; + int nri; + gmx_bool bFEP_i, bFEP_i_all; + + if (nbl_ci->cj_ind_end == nbl_ci->cj_ind_start) + { + /* Empty list */ + return; + } + + ci = nbl_ci->ci; + + cj_ind_start = nbl_ci->cj_ind_start; + cj_ind_end = nbl_ci->cj_ind_end; + + /* In worst case we have alternating energy groups + * and create #atom-pair lists, which means we need the size + * of a cluster pair (na_ci*na_cj) times the number of cj's. + */ + nri_max = nbl->na_ci*nbl->na_cj*(cj_ind_end - cj_ind_start); + if (nlist->nri + nri_max > nlist->maxnri) + { + nlist->maxnri = over_alloc_large(nlist->nri + nri_max); + reallocate_nblist(nlist); + } + + ngid = nbat->nenergrp; + + if (static_cast(ngid*gridj->na_cj) > sizeof(gid_cj)*8) + { + gmx_fatal(FARGS, "The Verlet scheme with %dx%d kernels and free-energy only supports up to %d energy groups", + gridi->na_c, gridj->na_cj, (sizeof(gid_cj)*8)/gridj->na_cj); + } + + egp_shift = nbat->neg_2log; + egp_mask = (1<neg_2log) - 1; + + /* Loop over the atoms in the i sub-cell */ + bFEP_i_all = TRUE; + for (int i = 0; i < nbl->na_ci; i++) + { + ind_i = ci*nbl->na_ci + i; + ai = nbs->a[ind_i]; + if (ai >= 0) + { + nri = nlist->nri; + nlist->jindex[nri+1] = nlist->jindex[nri]; + nlist->iinr[nri] = ai; + /* The actual energy group pair index is set later */ + nlist->gid[nri] = 0; + nlist->shift[nri] = nbl_ci->shift & NBNXN_CI_SHIFT; + + bFEP_i = gridi->fep[ci - gridi->cell0] & (1 << i); + + bFEP_i_all = bFEP_i_all && bFEP_i; + + if ((nlist->nrj + cj_ind_end - cj_ind_start)*nbl->na_cj > nlist->maxnrj) + { + nlist->maxnrj = over_alloc_small((nlist->nrj + cj_ind_end - cj_ind_start)*nbl->na_cj); + srenew(nlist->jjnr, nlist->maxnrj); + srenew(nlist->excl_fep, nlist->maxnrj); + } + + if (ngid > 1) + { + gid_i = (nbat->energrp[ci] >> (egp_shift*i)) & egp_mask; + } + + for (int cj_ind = cj_ind_start; cj_ind < cj_ind_end; cj_ind++) + { + unsigned int fep_cj; + + cja = nbl->cj[cj_ind].cj; + + if (gridj->na_cj == gridj->na_c) + { + cjr = cja - gridj->cell0; + fep_cj = gridj->fep[cjr]; + if (ngid > 1) + { + gid_cj = nbat->energrp[cja]; + } + } + else if (2*gridj->na_cj == gridj->na_c) + { + cjr = cja - gridj->cell0*2; + /* Extract half of the ci fep/energrp mask */ + fep_cj = (gridj->fep[cjr>>1] >> ((cjr&1)*gridj->na_cj)) & ((1<na_cj) - 1); + if (ngid > 1) + { + gid_cj = nbat->energrp[cja>>1] >> ((cja&1)*gridj->na_cj*egp_shift) & ((1<<(gridj->na_cj*egp_shift)) - 1); + } + } + else + { + cjr = cja - (gridj->cell0>>1); + /* Combine two ci fep masks/energrp */ + fep_cj = gridj->fep[cjr*2] + (gridj->fep[cjr*2+1] << gridj->na_c); + if (ngid > 1) + { + gid_cj = nbat->energrp[cja*2] + (nbat->energrp[cja*2+1] << (gridj->na_c*egp_shift)); + } + } + + if (bFEP_i || fep_cj != 0) + { + for (int j = 0; j < nbl->na_cj; j++) + { + /* Is this interaction perturbed and not excluded? */ + ind_j = cja*nbl->na_cj + j; + aj = nbs->a[ind_j]; + if (aj >= 0 && + (bFEP_i || (fep_cj & (1 << j))) && + (!bDiagRemoved || ind_j >= ind_i)) + { + if (ngid > 1) + { + gid_j = (gid_cj >> (j*egp_shift)) & egp_mask; + gid = GID(gid_i, gid_j, ngid); + + if (nlist->nrj > nlist->jindex[nri] && + nlist->gid[nri] != gid) + { + /* Energy group pair changed: new list */ + fep_list_new_nri_copy(nlist); + nri = nlist->nri; + } + nlist->gid[nri] = gid; + } + + if (nlist->nrj - nlist->jindex[nri] >= max_nrj_fep) + { + fep_list_new_nri_copy(nlist); + nri = nlist->nri; + } + + /* Add it to the FEP list */ + nlist->jjnr[nlist->nrj] = aj; + nlist->excl_fep[nlist->nrj] = (nbl->cj[cj_ind].excl >> (i*nbl->na_cj + j)) & 1; + nlist->nrj++; + + /* Exclude it from the normal list. + * Note that the charge has been set to zero, + * but we need to avoid 0/0, as perturbed atoms + * can be on top of each other. + */ + nbl->cj[cj_ind].excl &= ~(1U << (i*nbl->na_cj + j)); + } + } + } + } + + if (nlist->nrj > nlist->jindex[nri]) + { + /* Actually add this new, non-empty, list */ + nlist->nri++; + nlist->jindex[nlist->nri] = nlist->nrj; + } + } + } + + if (bFEP_i_all) + { + /* All interactions are perturbed, we can skip this entry */ + nbl_ci->cj_ind_end = cj_ind_start; + } +} + +/* Return the index of atom a within a cluster */ +static gmx_inline int cj_mod_cj4(int cj) +{ + return cj & (nbnxn_gpu_jgroup_size - 1); +} + +/* Convert a j-cluster to a cj4 group */ +static gmx_inline int cj_to_cj4(int cj) +{ + return cj/nbnxn_gpu_jgroup_size; +} + +/* Return the index of an j-atom within a warp */ +static gmx_inline int a_mod_wj(int a) +{ + return a & (nbnxn_gpu_cluster_size/nbnxn_gpu_clusterpair_split - 1); +} + +/* As make_fep_list above, but for super/sub lists. */ +static void make_fep_list_supersub(const nbnxn_search_t nbs, + const nbnxn_atomdata_t *nbat, + nbnxn_pairlist_t *nbl, + gmx_bool bDiagRemoved, + const nbnxn_sci_t *nbl_sci, + real shx, + real shy, + real shz, + real rlist_fep2, + const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj, + t_nblist *nlist) +{ + int sci, cj4_ind_start, cj4_ind_end, cjr; + int nri_max; + int c_abs; + int ind_i, ind_j, ai, aj; + int nri; + gmx_bool bFEP_i; + real xi, yi, zi; + const nbnxn_cj4_t *cj4; + + if (nbl_sci->cj4_ind_end == nbl_sci->cj4_ind_start) + { + /* Empty list */ + return; + } + + sci = nbl_sci->sci; + + cj4_ind_start = nbl_sci->cj4_ind_start; + cj4_ind_end = nbl_sci->cj4_ind_end; + + /* Here we process one super-cell, max #atoms na_sc, versus a list + * cj4 entries, each with max nbnxn_gpu_jgroup_size cj's, each + * of size na_cj atoms. + * On the GPU we don't support energy groups (yet). + * So for each of the na_sc i-atoms, we need max one FEP list + * for each max_nrj_fep j-atoms. + */ + nri_max = nbl->na_sc*nbl->na_cj*(1 + ((cj4_ind_end - cj4_ind_start)*nbnxn_gpu_jgroup_size)/max_nrj_fep); + if (nlist->nri + nri_max > nlist->maxnri) + { + nlist->maxnri = over_alloc_large(nlist->nri + nri_max); + reallocate_nblist(nlist); + } + + /* Loop over the atoms in the i super-cluster */ + for (int c = 0; c < gpu_ncluster_per_cell; c++) + { + c_abs = sci*gpu_ncluster_per_cell + c; + + for (int i = 0; i < nbl->na_ci; i++) + { + ind_i = c_abs*nbl->na_ci + i; + ai = nbs->a[ind_i]; + if (ai >= 0) + { + nri = nlist->nri; + nlist->jindex[nri+1] = nlist->jindex[nri]; + nlist->iinr[nri] = ai; + /* With GPUs, energy groups are not supported */ + nlist->gid[nri] = 0; + nlist->shift[nri] = nbl_sci->shift & NBNXN_CI_SHIFT; + + bFEP_i = (gridi->fep[c_abs - gridi->cell0*gpu_ncluster_per_cell] & (1 << i)); + + xi = nbat->x[ind_i*nbat->xstride+XX] + shx; + yi = nbat->x[ind_i*nbat->xstride+YY] + shy; + zi = nbat->x[ind_i*nbat->xstride+ZZ] + shz; + + if ((nlist->nrj + cj4_ind_end - cj4_ind_start)*nbnxn_gpu_jgroup_size*nbl->na_cj > nlist->maxnrj) + { + nlist->maxnrj = over_alloc_small((nlist->nrj + cj4_ind_end - cj4_ind_start)*nbnxn_gpu_jgroup_size*nbl->na_cj); + srenew(nlist->jjnr, nlist->maxnrj); + srenew(nlist->excl_fep, nlist->maxnrj); + } + + for (int cj4_ind = cj4_ind_start; cj4_ind < cj4_ind_end; cj4_ind++) + { + cj4 = &nbl->cj4[cj4_ind]; + + for (int gcj = 0; gcj < nbnxn_gpu_jgroup_size; gcj++) + { + unsigned int fep_cj; + + if ((cj4->imei[0].imask & (1U << (gcj*gpu_ncluster_per_cell + c))) == 0) + { + /* Skip this ci for this cj */ + continue; + } + + cjr = cj4->cj[gcj] - gridj->cell0*gpu_ncluster_per_cell; + + fep_cj = gridj->fep[cjr]; + + if (bFEP_i || fep_cj != 0) + { + for (int j = 0; j < nbl->na_cj; j++) + { + /* Is this interaction perturbed and not excluded? */ + ind_j = (gridj->cell0*gpu_ncluster_per_cell + cjr)*nbl->na_cj + j; + aj = nbs->a[ind_j]; + if (aj >= 0 && + (bFEP_i || (fep_cj & (1 << j))) && + (!bDiagRemoved || ind_j >= ind_i)) + { + nbnxn_excl_t *excl; + int excl_pair; + unsigned int excl_bit; + real dx, dy, dz; + + get_nbl_exclusions_1(nbl, cj4_ind, j>>2, &excl); + + excl_pair = a_mod_wj(j)*nbl->na_ci + i; + excl_bit = (1U << (gcj*gpu_ncluster_per_cell + c)); + + dx = nbat->x[ind_j*nbat->xstride+XX] - xi; + dy = nbat->x[ind_j*nbat->xstride+YY] - yi; + dz = nbat->x[ind_j*nbat->xstride+ZZ] - zi; + + /* The unpruned GPU list has more than 2/3 + * of the atom pairs beyond rlist. Using + * this list will cause a lot of overhead + * in the CPU FEP kernels, especially + * relative to the fast GPU kernels. + * So we prune the FEP list here. + */ + if (dx*dx + dy*dy + dz*dz < rlist_fep2) + { + if (nlist->nrj - nlist->jindex[nri] >= max_nrj_fep) + { + fep_list_new_nri_copy(nlist); + nri = nlist->nri; + } + + /* Add it to the FEP list */ + nlist->jjnr[nlist->nrj] = aj; + nlist->excl_fep[nlist->nrj] = (excl->pair[excl_pair] & excl_bit) ? 1 : 0; + nlist->nrj++; + } + + /* Exclude it from the normal list. + * Note that the charge and LJ parameters have + * been set to zero, but we need to avoid 0/0, + * as perturbed atoms can be on top of each other. + */ + excl->pair[excl_pair] &= ~excl_bit; + } + } + + /* Note that we could mask out this pair in imask + * if all i- and/or all j-particles are perturbed. + * But since the perturbed pairs on the CPU will + * take an order of magnitude more time, the GPU + * will finish before the CPU and there is no gain. + */ + } + } + } + + if (nlist->nrj > nlist->jindex[nri]) + { + /* Actually add this new, non-empty, list */ + nlist->nri++; + nlist->jindex[nlist->nri] = nlist->nrj; + } + } + } + } +} + +/* Set all atom-pair exclusions from the topology stored in excl + * as masks in the pair-list for i-super-cell entry nbl_sci + */ +static void set_sci_top_excls(const nbnxn_search_t nbs, + nbnxn_pairlist_t *nbl, + gmx_bool diagRemoved, + int na_c_2log, + const nbnxn_sci_t *nbl_sci, + const t_blocka *excl) +{ + const int *cell; + int na_c; + int sci; + int cj_ind_first, cj_ind_last; + int cj_first, cj_last; + int ndirect; + int ai, aj, si, ge, se; + int found, cj_ind_0, cj_ind_1, cj_ind_m; + int cj_m; + nbnxn_excl_t *nbl_excl; + int inner_i, inner_e, w; + + cell = nbs->cell; + + na_c = nbl->na_ci; + + if (nbl_sci->cj4_ind_end == nbl_sci->cj4_ind_start) + { + /* Empty list */ + return; + } + + sci = nbl_sci->sci; + + cj_ind_first = nbl_sci->cj4_ind_start*nbnxn_gpu_jgroup_size; + cj_ind_last = nbl->work->cj_ind - 1; + + cj_first = nbl->cj4[nbl_sci->cj4_ind_start].cj[0]; + cj_last = nbl_cj(nbl, cj_ind_last); + + /* Determine how many contiguous j-clusters we have starting + * from the first i-cluster. This number can be used to directly + * calculate j-cluster indices for excluded atoms. + */ + ndirect = 0; + while (cj_ind_first + ndirect <= cj_ind_last && + nbl_cj(nbl, cj_ind_first+ndirect) == sci*gpu_ncluster_per_cell + ndirect) + { + ndirect++; + } + + /* Loop over the atoms in the i super-cell */ + for (int i = 0; i < nbl->na_sc; i++) + { + ai = nbs->a[sci*nbl->na_sc+i]; + if (ai >= 0) + { + si = (i>>na_c_2log); + + /* Loop over the topology-based exclusions for this i-atom */ + for (int eind = excl->index[ai]; eind < excl->index[ai+1]; eind++) + { + aj = excl->a[eind]; + + if (aj == ai) + { + /* The self exclusion are already set, save some time */ + continue; + } + + ge = cell[aj]; + + /* Without shifts we only calculate interactions j>i + * for one-way pair-lists. + */ + if (diagRemoved && ge <= sci*nbl->na_sc + i) + { + continue; + } + + se = ge>>na_c_2log; + /* Could the cluster se be in our list? */ + if (se >= cj_first && se <= cj_last) + { + if (se < cj_first + ndirect) + { + /* We can calculate cj_ind directly from se */ + found = cj_ind_first + se - cj_first; + } + else + { + /* Search for se using bisection */ + found = -1; + cj_ind_0 = cj_ind_first + ndirect; + cj_ind_1 = cj_ind_last + 1; + while (found == -1 && cj_ind_0 < cj_ind_1) + { + cj_ind_m = (cj_ind_0 + cj_ind_1)>>1; + + cj_m = nbl_cj(nbl, cj_ind_m); + + if (se == cj_m) + { + found = cj_ind_m; + } + else if (se < cj_m) + { + cj_ind_1 = cj_ind_m; + } + else + { + cj_ind_0 = cj_ind_m + 1; + } + } + } + + if (found >= 0) + { + inner_i = i - si*na_c; + inner_e = ge - se*na_c; + + if (nbl_imask0(nbl, found) & (1U << (cj_mod_cj4(found)*gpu_ncluster_per_cell + si))) + { + w = (inner_e >> 2); + + get_nbl_exclusions_1(nbl, cj_to_cj4(found), w, &nbl_excl); + + nbl_excl->pair[a_mod_wj(inner_e)*nbl->na_ci+inner_i] &= + ~(1U << (cj_mod_cj4(found)*gpu_ncluster_per_cell + si)); + } + } + } + } + } + } +} + +/* Reallocate the simple ci list for at least n entries */ +static void nb_realloc_ci(nbnxn_pairlist_t *nbl, int n) +{ + nbl->ci_nalloc = over_alloc_small(n); + nbnxn_realloc_void((void **)&nbl->ci, + nbl->nci*sizeof(*nbl->ci), + nbl->ci_nalloc*sizeof(*nbl->ci), + nbl->alloc, nbl->free); +} + +/* Reallocate the super-cell sci list for at least n entries */ +static void nb_realloc_sci(nbnxn_pairlist_t *nbl, int n) +{ + nbl->sci_nalloc = over_alloc_small(n); + nbnxn_realloc_void((void **)&nbl->sci, + nbl->nsci*sizeof(*nbl->sci), + nbl->sci_nalloc*sizeof(*nbl->sci), + nbl->alloc, nbl->free); +} + +/* Make a new ci entry at index nbl->nci */ +static void new_ci_entry(nbnxn_pairlist_t *nbl, int ci, int shift, int flags) +{ + if (nbl->nci + 1 > nbl->ci_nalloc) + { + nb_realloc_ci(nbl, nbl->nci+1); + } + nbl->ci[nbl->nci].ci = ci; + nbl->ci[nbl->nci].shift = shift; + /* Store the interaction flags along with the shift */ + nbl->ci[nbl->nci].shift |= flags; + nbl->ci[nbl->nci].cj_ind_start = nbl->ncj; + nbl->ci[nbl->nci].cj_ind_end = nbl->ncj; +} + +/* Make a new sci entry at index nbl->nsci */ +static void new_sci_entry(nbnxn_pairlist_t *nbl, int sci, int shift) +{ + if (nbl->nsci + 1 > nbl->sci_nalloc) + { + nb_realloc_sci(nbl, nbl->nsci+1); + } + nbl->sci[nbl->nsci].sci = sci; + nbl->sci[nbl->nsci].shift = shift; + nbl->sci[nbl->nsci].cj4_ind_start = nbl->ncj4; + nbl->sci[nbl->nsci].cj4_ind_end = nbl->ncj4; +} + +/* Sort the simple j-list cj on exclusions. + * Entries with exclusions will all be sorted to the beginning of the list. + */ +static void sort_cj_excl(nbnxn_cj_t *cj, int ncj, + nbnxn_list_work_t *work) +{ + int jnew; + + if (ncj > work->cj_nalloc) + { + work->cj_nalloc = over_alloc_large(ncj); + srenew(work->cj, work->cj_nalloc); + } + + /* Make a list of the j-cells involving exclusions */ + jnew = 0; + for (int j = 0; j < ncj; j++) + { + if (cj[j].excl != NBNXN_INTERACTION_MASK_ALL) + { + work->cj[jnew++] = cj[j]; + } + } + /* Check if there are exclusions at all or not just the first entry */ + if (!((jnew == 0) || + (jnew == 1 && cj[0].excl != NBNXN_INTERACTION_MASK_ALL))) + { + for (int j = 0; j < ncj; j++) + { + if (cj[j].excl == NBNXN_INTERACTION_MASK_ALL) + { + work->cj[jnew++] = cj[j]; + } + } + for (int j = 0; j < ncj; j++) + { + cj[j] = work->cj[j]; + } + } +} + +/* Close this simple list i entry */ +static void close_ci_entry_simple(nbnxn_pairlist_t *nbl) +{ + int jlen; + + /* All content of the new ci entry have already been filled correctly, + * we only need to increase the count here (for non empty lists). + */ + jlen = nbl->ci[nbl->nci].cj_ind_end - nbl->ci[nbl->nci].cj_ind_start; + if (jlen > 0) + { + sort_cj_excl(nbl->cj+nbl->ci[nbl->nci].cj_ind_start, jlen, nbl->work); + + /* The counts below are used for non-bonded pair/flop counts + * and should therefore match the available kernel setups. + */ + if (!(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_COUL(0))) + { + nbl->work->ncj_noq += jlen; + } + else if ((nbl->ci[nbl->nci].shift & NBNXN_CI_HALF_LJ(0)) || + !(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_LJ(0))) + { + nbl->work->ncj_hlj += jlen; + } + + nbl->nci++; + } +} + +/* Split sci entry for load balancing on the GPU. + * Splitting ensures we have enough lists to fully utilize the whole GPU. + * With progBal we generate progressively smaller lists, which improves + * load balancing. As we only know the current count on our own thread, + * we will need to estimate the current total amount of i-entries. + * As the lists get concatenated later, this estimate depends + * both on nthread and our own thread index. + */ +static void split_sci_entry(nbnxn_pairlist_t *nbl, + int nsp_target_av, + gmx_bool progBal, int nsp_tot_est, + int thread, int nthread) +{ + int nsp_est; + int nsp_max; + int cj4_start, cj4_end, j4len; + int sci; + int nsp, nsp_sci, nsp_cj4, nsp_cj4_e, nsp_cj4_p; + + if (progBal) + { + /* Estimate the total numbers of ci's of the nblist combined + * over all threads using the target number of ci's. + */ + nsp_est = (nsp_tot_est*thread)/nthread + nbl->nci_tot; + + /* The first ci blocks should be larger, to avoid overhead. + * The last ci blocks should be smaller, to improve load balancing. + * The factor 3/2 makes the first block 3/2 times the target average + * and ensures that the total number of blocks end up equal to + * that with of equally sized blocks of size nsp_target_av. + */ + nsp_max = nsp_target_av*nsp_tot_est*3/(2*(nsp_est + nsp_tot_est)); + } + else + { + nsp_max = nsp_target_av; + } + + /* Since nsp_max is a maximum/cut-off (this avoids high outliers, + * which lead to load imbalance), not an average, we add half the + * number of pairs in a cj4 block to get the average about right. + */ + nsp_max += gpu_ncluster_per_cell*nbnxn_gpu_jgroup_size/2; + + cj4_start = nbl->sci[nbl->nsci-1].cj4_ind_start; + cj4_end = nbl->sci[nbl->nsci-1].cj4_ind_end; + j4len = cj4_end - cj4_start; + + if (j4len > 1 && j4len*gpu_ncluster_per_cell*nbnxn_gpu_jgroup_size > nsp_max) + { + /* Remove the last ci entry and process the cj4's again */ + nbl->nsci -= 1; + + sci = nbl->nsci; + nsp = 0; + nsp_sci = 0; + nsp_cj4_e = 0; + nsp_cj4 = 0; + for (int cj4 = cj4_start; cj4 < cj4_end; cj4++) + { + nsp_cj4_p = nsp_cj4; + /* Count the number of cluster pairs in this cj4 group */ + nsp_cj4 = 0; + for (int p = 0; p < gpu_ncluster_per_cell*nbnxn_gpu_jgroup_size; p++) + { + nsp_cj4 += (nbl->cj4[cj4].imei[0].imask >> p) & 1; + } + + /* Check if we should split at this cj4 to get a list of size nsp */ + if (nsp > 0 && nsp + nsp_cj4 > nsp_max) + { + /* Split the list at cj4 */ + nbl->sci[sci].cj4_ind_end = cj4; + /* Create a new sci entry */ + sci++; + nbl->nsci++; + if (nbl->nsci+1 > nbl->sci_nalloc) + { + nb_realloc_sci(nbl, nbl->nsci+1); + } + nbl->sci[sci].sci = nbl->sci[nbl->nsci-1].sci; + nbl->sci[sci].shift = nbl->sci[nbl->nsci-1].shift; + nbl->sci[sci].cj4_ind_start = cj4; + nsp_sci = nsp; + nsp_cj4_e = nsp_cj4_p; + nsp = 0; + } + nsp += nsp_cj4; + } + + /* Put the remaining cj4's in the last sci entry */ + nbl->sci[sci].cj4_ind_end = cj4_end; + + /* Possibly balance out the last two sci's + * by moving the last cj4 of the second last sci. + */ + if (nsp_sci - nsp_cj4_e >= nsp + nsp_cj4_e) + { + nbl->sci[sci-1].cj4_ind_end--; + nbl->sci[sci].cj4_ind_start--; + } + + nbl->nsci++; + } +} + +/* Clost this super/sub list i entry */ +static void close_ci_entry_supersub(nbnxn_pairlist_t *nbl, + int nsp_max_av, + gmx_bool progBal, int nsp_tot_est, + int thread, int nthread) +{ + /* All content of the new ci entry have already been filled correctly, + * we only need to increase the count here (for non empty lists). + */ + int j4len = nbl->sci[nbl->nsci].cj4_ind_end - nbl->sci[nbl->nsci].cj4_ind_start; + if (j4len > 0) + { + /* We can only have complete blocks of 4 j-entries in a list, + * so round the count up before closing. + */ + nbl->ncj4 = (nbl->work->cj_ind + nbnxn_gpu_jgroup_size - 1)/nbnxn_gpu_jgroup_size; + nbl->work->cj_ind = nbl->ncj4*nbnxn_gpu_jgroup_size; + + nbl->nsci++; + + if (nsp_max_av > 0) + { + /* Measure the size of the new entry and potentially split it */ + split_sci_entry(nbl, nsp_max_av, progBal, nsp_tot_est, + thread, nthread); + } + } +} + +/* Syncs the working array before adding another grid pair to the list */ +static void sync_work(nbnxn_pairlist_t *nbl) +{ + if (!nbl->bSimple) + { + nbl->work->cj_ind = nbl->ncj4*nbnxn_gpu_jgroup_size; + nbl->work->cj4_init = nbl->ncj4; + } +} + +/* Clears an nbnxn_pairlist_t data structure */ +static void clear_pairlist(nbnxn_pairlist_t *nbl) +{ + nbl->nci = 0; + nbl->nsci = 0; + nbl->ncj = 0; + nbl->ncj4 = 0; + nbl->nci_tot = 0; + nbl->nexcl = 1; + + nbl->work->ncj_noq = 0; + nbl->work->ncj_hlj = 0; +} + +/* Clears a group scheme pair list */ +static void clear_pairlist_fep(t_nblist *nl) +{ + nl->nri = 0; + nl->nrj = 0; + if (nl->jindex == NULL) + { + snew(nl->jindex, 1); + } + nl->jindex[0] = 0; +} + +/* Sets a simple list i-cell bounding box, including PBC shift */ +static gmx_inline void set_icell_bb_simple(const nbnxn_bb_t *bb, int ci, + real shx, real shy, real shz, + nbnxn_bb_t *bb_ci) +{ + bb_ci->lower[BB_X] = bb[ci].lower[BB_X] + shx; + bb_ci->lower[BB_Y] = bb[ci].lower[BB_Y] + shy; + bb_ci->lower[BB_Z] = bb[ci].lower[BB_Z] + shz; + bb_ci->upper[BB_X] = bb[ci].upper[BB_X] + shx; + bb_ci->upper[BB_Y] = bb[ci].upper[BB_Y] + shy; + bb_ci->upper[BB_Z] = bb[ci].upper[BB_Z] + shz; +} + +#ifdef NBNXN_BBXXXX +/* Sets a super-cell and sub cell bounding boxes, including PBC shift */ +static void set_icell_bbxxxx_supersub(const float *bb, int ci, + real shx, real shy, real shz, + float *bb_ci) +{ + int ia = ci*(gpu_ncluster_per_cell >> STRIDE_PBB_2LOG)*NNBSBB_XXXX; + for (int m = 0; m < (gpu_ncluster_per_cell >> STRIDE_PBB_2LOG)*NNBSBB_XXXX; m += NNBSBB_XXXX) + { + for (int i = 0; i < STRIDE_PBB; i++) + { + bb_ci[m+0*STRIDE_PBB+i] = bb[ia+m+0*STRIDE_PBB+i] + shx; + bb_ci[m+1*STRIDE_PBB+i] = bb[ia+m+1*STRIDE_PBB+i] + shy; + bb_ci[m+2*STRIDE_PBB+i] = bb[ia+m+2*STRIDE_PBB+i] + shz; + bb_ci[m+3*STRIDE_PBB+i] = bb[ia+m+3*STRIDE_PBB+i] + shx; + bb_ci[m+4*STRIDE_PBB+i] = bb[ia+m+4*STRIDE_PBB+i] + shy; + bb_ci[m+5*STRIDE_PBB+i] = bb[ia+m+5*STRIDE_PBB+i] + shz; + } + } +} +#endif + +/* Sets a super-cell and sub cell bounding boxes, including PBC shift */ +static void set_icell_bb_supersub(const nbnxn_bb_t *bb, int ci, + real shx, real shy, real shz, + nbnxn_bb_t *bb_ci) +{ + for (int i = 0; i < gpu_ncluster_per_cell; i++) + { + set_icell_bb_simple(bb, ci*gpu_ncluster_per_cell+i, + shx, shy, shz, + &bb_ci[i]); + } +} + +/* Copies PBC shifted i-cell atom coordinates x,y,z to working array */ +static void icell_set_x_simple(int ci, + real shx, real shy, real shz, + int stride, const real *x, + nbnxn_list_work_t *work) +{ + int ia = ci*NBNXN_CPU_CLUSTER_I_SIZE; + + for (int i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE; i++) + { + work->x_ci[i*STRIDE_XYZ+XX] = x[(ia+i)*stride+XX] + shx; + work->x_ci[i*STRIDE_XYZ+YY] = x[(ia+i)*stride+YY] + shy; + work->x_ci[i*STRIDE_XYZ+ZZ] = x[(ia+i)*stride+ZZ] + shz; + } +} + +/* Copies PBC shifted super-cell atom coordinates x,y,z to working array */ +static void icell_set_x_supersub(int ci, + real shx, real shy, real shz, + int stride, const real *x, + nbnxn_list_work_t *work) +{ +#ifndef NBNXN_SEARCH_BB_SIMD4 + + real * x_ci = work->x_ci; + + int ia = ci*gpu_ncluster_per_cell*nbnxn_gpu_cluster_size; + for (int i = 0; i < gpu_ncluster_per_cell*nbnxn_gpu_cluster_size; i++) + { + x_ci[i*DIM + XX] = x[(ia+i)*stride + XX] + shx; + x_ci[i*DIM + YY] = x[(ia+i)*stride + YY] + shy; + x_ci[i*DIM + ZZ] = x[(ia+i)*stride + ZZ] + shz; + } + +#else /* !NBNXN_SEARCH_BB_SIMD4 */ + + real * x_ci = work->x_ci_simd; + + for (int si = 0; si < gpu_ncluster_per_cell; si++) + { + for (int i = 0; i < nbnxn_gpu_cluster_size; i += GMX_SIMD4_WIDTH) + { + int io = si*nbnxn_gpu_cluster_size + i; + int ia = ci*gpu_ncluster_per_cell*nbnxn_gpu_cluster_size + io; + for (int j = 0; j < GMX_SIMD4_WIDTH; j++) + { + x_ci[io*DIM + j + XX*GMX_SIMD4_WIDTH] = x[(ia + j)*stride + XX] + shx; + x_ci[io*DIM + j + YY*GMX_SIMD4_WIDTH] = x[(ia + j)*stride + YY] + shy; + x_ci[io*DIM + j + ZZ*GMX_SIMD4_WIDTH] = x[(ia + j)*stride + ZZ] + shz; + } + } + } + +#endif /* !NBNXN_SEARCH_BB_SIMD4 */ +} + +static real minimum_subgrid_size_xy(const nbnxn_grid_t *grid) +{ + if (grid->bSimple) + { + return std::min(grid->sx, grid->sy); + } + else + { + return std::min(grid->sx/gpu_ncluster_per_cell_x, + grid->sy/gpu_ncluster_per_cell_y); + } +} + +static real effective_buffer_1x1_vs_MxN(const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj) +{ + const real eff_1x1_buffer_fac_overest = 0.1; + + /* Determine an atom-pair list cut-off buffer size for atom pairs, + * to be added to rlist (including buffer) used for MxN. + * This is for converting an MxN list to a 1x1 list. This means we can't + * use the normal buffer estimate, as we have an MxN list in which + * some atom pairs beyond rlist are missing. We want to capture + * the beneficial effect of buffering by extra pairs just outside rlist, + * while removing the useless pairs that are further away from rlist. + * (Also the buffer could have been set manually not using the estimate.) + * This buffer size is an overestimate. + * We add 10% of the smallest grid sub-cell dimensions. + * Note that the z-size differs per cell and we don't use this, + * so we overestimate. + * With PME, the 10% value gives a buffer that is somewhat larger + * than the effective buffer with a tolerance of 0.005 kJ/mol/ps. + * Smaller tolerances or using RF lead to a smaller effective buffer, + * so 10% gives a safe overestimate. + */ + return eff_1x1_buffer_fac_overest*(minimum_subgrid_size_xy(gridi) + + minimum_subgrid_size_xy(gridj)); +} + +/* Clusters at the cut-off only increase rlist by 60% of their size */ +static real nbnxn_rlist_inc_outside_fac = 0.6; + +/* Due to the cluster size the effective pair-list is longer than + * that of a simple atom pair-list. This function gives the extra distance. + */ +real nbnxn_get_rlist_effective_inc(int cluster_size_j, real atom_density) +{ + int cluster_size_i; + real vol_inc_i, vol_inc_j; + + /* We should get this from the setup, but currently it's the same for + * all setups, including GPUs. + */ + cluster_size_i = NBNXN_CPU_CLUSTER_I_SIZE; + + vol_inc_i = (cluster_size_i - 1)/atom_density; + vol_inc_j = (cluster_size_j - 1)/atom_density; + + return nbnxn_rlist_inc_outside_fac*std::cbrt(vol_inc_i + vol_inc_j); +} + +/* Estimates the interaction volume^2 for non-local interactions */ +static real nonlocal_vol2(const struct gmx_domdec_zones_t *zones, rvec ls, real r) +{ + real cl, ca, za; + real vold_est; + real vol2_est_tot; + + vol2_est_tot = 0; + + /* Here we simply add up the volumes of 1, 2 or 3 1D decomposition + * not home interaction volume^2. As these volumes are not additive, + * this is an overestimate, but it would only be significant in the limit + * of small cells, where we anyhow need to split the lists into + * as small parts as possible. + */ + + for (int z = 0; z < zones->n; z++) + { + if (zones->shift[z][XX] + zones->shift[z][YY] + zones->shift[z][ZZ] == 1) + { + cl = 0; + ca = 1; + za = 1; + for (int d = 0; d < DIM; d++) + { + if (zones->shift[z][d] == 0) + { + cl += 0.5*ls[d]; + ca *= ls[d]; + za *= zones->size[z].x1[d] - zones->size[z].x0[d]; + } + } + + /* 4 octants of a sphere */ + vold_est = 0.25*M_PI*r*r*r*r; + /* 4 quarter pie slices on the edges */ + vold_est += 4*cl*M_PI/6.0*r*r*r; + /* One rectangular volume on a face */ + vold_est += ca*0.5*r*r; + + vol2_est_tot += vold_est*za; + } + } + + return vol2_est_tot; +} + +/* Estimates the average size of a full j-list for super/sub setup */ +static void get_nsubpair_target(const nbnxn_search_t nbs, + int iloc, + real rlist, + int min_ci_balanced, + int *nsubpair_target, + int *nsubpair_tot_est) +{ + /* The target value of 36 seems to be the optimum for Kepler. + * Maxwell is less sensitive to the exact value. + */ + const int nsubpair_target_min = 36; + const nbnxn_grid_t *grid; + rvec ls; + real xy_diag2, r_eff_sup, vol_est, nsp_est, nsp_est_nl; + + grid = &nbs->grid[0]; + + if (min_ci_balanced <= 0 || grid->nc >= min_ci_balanced || grid->nc == 0) + { + /* We don't need to balance the list sizes */ + *nsubpair_target = 0; + *nsubpair_tot_est = 0; + + return; + } + + ls[XX] = (grid->c1[XX] - grid->c0[XX])/(grid->ncx*gpu_ncluster_per_cell_x); + ls[YY] = (grid->c1[YY] - grid->c0[YY])/(grid->ncy*gpu_ncluster_per_cell_y); + ls[ZZ] = grid->na_c/(grid->atom_density*ls[XX]*ls[YY]); + + /* The average squared length of the diagonal of a sub cell */ + xy_diag2 = ls[XX]*ls[XX] + ls[YY]*ls[YY] + ls[ZZ]*ls[ZZ]; + + /* The formulas below are a heuristic estimate of the average nsj per si*/ + r_eff_sup = rlist + nbnxn_rlist_inc_outside_fac*gmx::square((grid->na_c - 1.0)/grid->na_c)*std::sqrt(xy_diag2/3); + + if (!nbs->DomDec || nbs->zones->n == 1) + { + nsp_est_nl = 0; + } + else + { + nsp_est_nl = + gmx::square(grid->atom_density/grid->na_c)* + nonlocal_vol2(nbs->zones, ls, r_eff_sup); + } + + if (LOCAL_I(iloc)) + { + /* Sub-cell interacts with itself */ + vol_est = ls[XX]*ls[YY]*ls[ZZ]; + /* 6/2 rectangular volume on the faces */ + vol_est += (ls[XX]*ls[YY] + ls[XX]*ls[ZZ] + ls[YY]*ls[ZZ])*r_eff_sup; + /* 12/2 quarter pie slices on the edges */ + vol_est += 2*(ls[XX] + ls[YY] + ls[ZZ])*0.25*M_PI*gmx::square(r_eff_sup); + /* 4 octants of a sphere */ + vol_est += 0.5*4.0/3.0*M_PI*gmx::power3(r_eff_sup); + + /* Estimate the number of cluster pairs as the local number of + * clusters times the volume they interact with times the density. + */ + nsp_est = grid->nsubc_tot*vol_est*grid->atom_density/grid->na_c; + + /* Subtract the non-local pair count */ + nsp_est -= nsp_est_nl; + + /* For small cut-offs nsp_est will be an underesimate. + * With DD nsp_est_nl is an overestimate so nsp_est can get negative. + * So to avoid too small or negative nsp_est we set a minimum of + * all cells interacting with all 3^3 direct neighbors (3^3-1)/2+1=14. + * This might be a slight overestimate for small non-periodic groups of + * atoms as will occur for a local domain with DD, but for small + * groups of atoms we'll anyhow be limited by nsubpair_target_min, + * so this overestimation will not matter. + */ + nsp_est = std::max(nsp_est, grid->nsubc_tot*static_cast(14)); + + if (debug) + { + fprintf(debug, "nsp_est local %5.1f non-local %5.1f\n", + nsp_est, nsp_est_nl); + } + } + else + { + nsp_est = nsp_est_nl; + } + + /* Thus the (average) maximum j-list size should be as follows. + * Since there is overhead, we shouldn't make the lists too small + * (and we can't chop up j-groups) so we use a minimum target size of 36. + */ + *nsubpair_target = std::max(nsubpair_target_min, + static_cast(nsp_est/min_ci_balanced + 0.5)); + *nsubpair_tot_est = static_cast(nsp_est); + + if (debug) + { + fprintf(debug, "nbl nsp estimate %.1f, nsubpair_target %d\n", + nsp_est, *nsubpair_target); + } +} + +/* Debug list print function */ +static void print_nblist_ci_cj(FILE *fp, const nbnxn_pairlist_t *nbl) +{ + for (int i = 0; i < nbl->nci; i++) + { + fprintf(fp, "ci %4d shift %2d ncj %3d\n", + nbl->ci[i].ci, nbl->ci[i].shift, + nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start); + + for (int j = nbl->ci[i].cj_ind_start; j < nbl->ci[i].cj_ind_end; j++) + { + fprintf(fp, " cj %5d imask %x\n", + nbl->cj[j].cj, + nbl->cj[j].excl); + } + } +} + +/* Debug list print function */ +static void print_nblist_sci_cj(FILE *fp, const nbnxn_pairlist_t *nbl) +{ + for (int i = 0; i < nbl->nsci; i++) + { + fprintf(fp, "ci %4d shift %2d ncj4 %2d\n", + nbl->sci[i].sci, nbl->sci[i].shift, + nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start); + + int ncp = 0; + for (int j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++) + { + for (int j = 0; j < nbnxn_gpu_jgroup_size; j++) + { + fprintf(fp, " sj %5d imask %x\n", + nbl->cj4[j4].cj[j], + nbl->cj4[j4].imei[0].imask); + for (int si = 0; si < gpu_ncluster_per_cell; si++) + { + if (nbl->cj4[j4].imei[0].imask & (1U << (j*gpu_ncluster_per_cell + si))) + { + ncp++; + } + } + } + } + fprintf(fp, "ci %4d shift %2d ncj4 %2d ncp %3d\n", + nbl->sci[i].sci, nbl->sci[i].shift, + nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start, + ncp); + } +} + +/* Combine pair lists *nbl generated on multiple threads nblc */ +static void combine_nblists(int nnbl, nbnxn_pairlist_t **nbl, + nbnxn_pairlist_t *nblc) +{ + int nsci, ncj4, nexcl; + + if (nblc->bSimple) + { + gmx_incons("combine_nblists does not support simple lists"); + } + + nsci = nblc->nsci; + ncj4 = nblc->ncj4; + nexcl = nblc->nexcl; + for (int i = 0; i < nnbl; i++) + { + nsci += nbl[i]->nsci; + ncj4 += nbl[i]->ncj4; + nexcl += nbl[i]->nexcl; + } + + if (nsci > nblc->sci_nalloc) + { + nb_realloc_sci(nblc, nsci); + } + if (ncj4 > nblc->cj4_nalloc) + { + nblc->cj4_nalloc = over_alloc_small(ncj4); + nbnxn_realloc_void((void **)&nblc->cj4, + nblc->ncj4*sizeof(*nblc->cj4), + nblc->cj4_nalloc*sizeof(*nblc->cj4), + nblc->alloc, nblc->free); + } + if (nexcl > nblc->excl_nalloc) + { + nblc->excl_nalloc = over_alloc_small(nexcl); + nbnxn_realloc_void((void **)&nblc->excl, + nblc->nexcl*sizeof(*nblc->excl), + nblc->excl_nalloc*sizeof(*nblc->excl), + nblc->alloc, nblc->free); + } + + /* Each thread should copy its own data to the combined arrays, + * as otherwise data will go back and forth between different caches. + */ +#if GMX_OPENMP && !(defined __clang_analyzer__) + // cppcheck-suppress unreadVariable + int nthreads = gmx_omp_nthreads_get(emntPairsearch); +#endif + +#pragma omp parallel for num_threads(nthreads) schedule(static) + for (int n = 0; n < nnbl; n++) + { + try + { + int sci_offset; + int cj4_offset; + int excl_offset; + const nbnxn_pairlist_t *nbli; + + /* Determine the offset in the combined data for our thread */ + sci_offset = nblc->nsci; + cj4_offset = nblc->ncj4; + excl_offset = nblc->nexcl; + + for (int i = 0; i < n; i++) + { + sci_offset += nbl[i]->nsci; + cj4_offset += nbl[i]->ncj4; + excl_offset += nbl[i]->nexcl; + } + + nbli = nbl[n]; + + for (int i = 0; i < nbli->nsci; i++) + { + nblc->sci[sci_offset+i] = nbli->sci[i]; + nblc->sci[sci_offset+i].cj4_ind_start += cj4_offset; + nblc->sci[sci_offset+i].cj4_ind_end += cj4_offset; + } + + for (int j4 = 0; j4 < nbli->ncj4; j4++) + { + nblc->cj4[cj4_offset+j4] = nbli->cj4[j4]; + nblc->cj4[cj4_offset+j4].imei[0].excl_ind += excl_offset; + nblc->cj4[cj4_offset+j4].imei[1].excl_ind += excl_offset; + } + + for (int j4 = 0; j4 < nbli->nexcl; j4++) + { + nblc->excl[excl_offset+j4] = nbli->excl[j4]; + } + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + } + + for (int n = 0; n < nnbl; n++) + { + nblc->nsci += nbl[n]->nsci; + nblc->ncj4 += nbl[n]->ncj4; + nblc->nci_tot += nbl[n]->nci_tot; + nblc->nexcl += nbl[n]->nexcl; + } +} + +static void balance_fep_lists(const nbnxn_search_t nbs, + nbnxn_pairlist_set_t *nbl_lists) +{ + int nnbl; + int nri_tot, nrj_tot, nrj_target; + int th_dest; + t_nblist *nbld; + + nnbl = nbl_lists->nnbl; + + if (nnbl == 1) + { + /* Nothing to balance */ + return; + } + + /* Count the total i-lists and pairs */ + nri_tot = 0; + nrj_tot = 0; + for (int th = 0; th < nnbl; th++) + { + nri_tot += nbl_lists->nbl_fep[th]->nri; + nrj_tot += nbl_lists->nbl_fep[th]->nrj; + } + + nrj_target = (nrj_tot + nnbl - 1)/nnbl; + + assert(gmx_omp_nthreads_get(emntNonbonded) == nnbl); + +#pragma omp parallel for schedule(static) num_threads(nnbl) + for (int th = 0; th < nnbl; th++) + { + try + { + t_nblist *nbl; + + nbl = nbs->work[th].nbl_fep; + + /* Note that here we allocate for the total size, instead of + * a per-thread esimate (which is hard to obtain). + */ + if (nri_tot > nbl->maxnri) + { + nbl->maxnri = over_alloc_large(nri_tot); + reallocate_nblist(nbl); + } + if (nri_tot > nbl->maxnri || nrj_tot > nbl->maxnrj) + { + nbl->maxnrj = over_alloc_small(nrj_tot); + srenew(nbl->jjnr, nbl->maxnrj); + srenew(nbl->excl_fep, nbl->maxnrj); + } + + clear_pairlist_fep(nbl); + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + } + + /* Loop over the source lists and assign and copy i-entries */ + th_dest = 0; + nbld = nbs->work[th_dest].nbl_fep; + for (int th = 0; th < nnbl; th++) + { + t_nblist *nbls; + + nbls = nbl_lists->nbl_fep[th]; + + for (int i = 0; i < nbls->nri; i++) + { + int nrj; + + /* The number of pairs in this i-entry */ + nrj = nbls->jindex[i+1] - nbls->jindex[i]; + + /* Decide if list th_dest is too large and we should procede + * to the next destination list. + */ + if (th_dest+1 < nnbl && nbld->nrj > 0 && + nbld->nrj + nrj - nrj_target > nrj_target - nbld->nrj) + { + th_dest++; + nbld = nbs->work[th_dest].nbl_fep; + } + + nbld->iinr[nbld->nri] = nbls->iinr[i]; + nbld->gid[nbld->nri] = nbls->gid[i]; + nbld->shift[nbld->nri] = nbls->shift[i]; + + for (int j = nbls->jindex[i]; j < nbls->jindex[i+1]; j++) + { + nbld->jjnr[nbld->nrj] = nbls->jjnr[j]; + nbld->excl_fep[nbld->nrj] = nbls->excl_fep[j]; + nbld->nrj++; + } + nbld->nri++; + nbld->jindex[nbld->nri] = nbld->nrj; + } + } + + /* Swap the list pointers */ + for (int th = 0; th < nnbl; th++) + { + t_nblist *nbl_tmp; + + nbl_tmp = nbl_lists->nbl_fep[th]; + nbl_lists->nbl_fep[th] = nbs->work[th].nbl_fep; + nbs->work[th].nbl_fep = nbl_tmp; + + if (debug) + { + fprintf(debug, "nbl_fep[%d] nri %4d nrj %4d\n", + th, + nbl_lists->nbl_fep[th]->nri, + nbl_lists->nbl_fep[th]->nrj); + } + } +} + +/* Returns the next ci to be processes by our thread */ +static gmx_bool next_ci(const nbnxn_grid_t *grid, + int conv, + int nth, int ci_block, + int *ci_x, int *ci_y, + int *ci_b, int *ci) +{ + (*ci_b)++; + (*ci)++; + + if (*ci_b == ci_block) + { + /* Jump to the next block assigned to this task */ + *ci += (nth - 1)*ci_block; + *ci_b = 0; + } + + if (*ci >= grid->nc*conv) + { + return FALSE; + } + + while (*ci >= grid->cxy_ind[*ci_x*grid->ncy + *ci_y + 1]*conv) + { + *ci_y += 1; + if (*ci_y == grid->ncy) + { + *ci_x += 1; + *ci_y = 0; + } + } + + return TRUE; +} + +/* Returns the distance^2 for which we put cell pairs in the list + * without checking atom pair distances. This is usually < rlist^2. + */ +static float boundingbox_only_distance2(const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj, + real rlist, + gmx_bool simple) +{ + /* If the distance between two sub-cell bounding boxes is less + * than this distance, do not check the distance between + * all particle pairs in the sub-cell, since then it is likely + * that the box pair has atom pairs within the cut-off. + * We use the nblist cut-off minus 0.5 times the average x/y diagonal + * spacing of the sub-cells. Around 40% of the checked pairs are pruned. + * Using more than 0.5 gains at most 0.5%. + * If forces are calculated more than twice, the performance gain + * in the force calculation outweighs the cost of checking. + * Note that with subcell lists, the atom-pair distance check + * is only performed when only 1 out of 8 sub-cells in within range, + * this is because the GPU is much faster than the cpu. + */ + real bbx, bby; + real rbb2; + + bbx = 0.5*(gridi->sx + gridj->sx); + bby = 0.5*(gridi->sy + gridj->sy); + if (!simple) + { + bbx /= gpu_ncluster_per_cell_x; + bby /= gpu_ncluster_per_cell_y; + } + + rbb2 = std::max(0.0, rlist - 0.5*std::sqrt(bbx*bbx + bby*bby)); + rbb2 = rbb2 * rbb2; + +#if !GMX_DOUBLE + return rbb2; +#else + return (float)((1+GMX_FLOAT_EPS)*rbb2); +#endif +} + +static int get_ci_block_size(const nbnxn_grid_t *gridi, + gmx_bool bDomDec, int nth) +{ + const int ci_block_enum = 5; + const int ci_block_denom = 11; + const int ci_block_min_atoms = 16; + int ci_block; + + /* Here we decide how to distribute the blocks over the threads. + * We use prime numbers to try to avoid that the grid size becomes + * a multiple of the number of threads, which would lead to some + * threads getting "inner" pairs and others getting boundary pairs, + * which in turns will lead to load imbalance between threads. + * Set the block size as 5/11/ntask times the average number of cells + * in a y,z slab. This should ensure a quite uniform distribution + * of the grid parts of the different thread along all three grid + * zone boundaries with 3D domain decomposition. At the same time + * the blocks will not become too small. + */ + ci_block = (gridi->nc*ci_block_enum)/(ci_block_denom*gridi->ncx*nth); + + /* Ensure the blocks are not too small: avoids cache invalidation */ + if (ci_block*gridi->na_sc < ci_block_min_atoms) + { + ci_block = (ci_block_min_atoms + gridi->na_sc - 1)/gridi->na_sc; + } + + /* Without domain decomposition + * or with less than 3 blocks per task, divide in nth blocks. + */ + if (!bDomDec || nth*3*ci_block > gridi->nc) + { + ci_block = (gridi->nc + nth - 1)/nth; + } + + if (ci_block > 1 && (nth - 1)*ci_block >= gridi->nc) + { + /* Some threads have no work. Although reducing the block size + * does not decrease the block count on the first few threads, + * with GPUs better mixing of "upper" cells that have more empty + * clusters results in a somewhat lower max load over all threads. + * Without GPUs the regime of so few atoms per thread is less + * performance relevant, but with 8-wide SIMD the same reasoning + * applies, since the pair list uses 4 i-atom "sub-clusters". + */ + ci_block--; + } + + return ci_block; +} + +/* Generates the part of pair-list nbl assigned to our thread */ +static void nbnxn_make_pairlist_part(const nbnxn_search_t nbs, + const nbnxn_grid_t *gridi, + const nbnxn_grid_t *gridj, + nbnxn_search_work_t *work, + const nbnxn_atomdata_t *nbat, + const t_blocka *excl, + real rlist, + int nb_kernel_type, + int ci_block, + gmx_bool bFBufferFlag, + int nsubpair_max, + gmx_bool progBal, + int nsubpair_tot_est, + int th, int nth, + nbnxn_pairlist_t *nbl, + t_nblist *nbl_fep) +{ + int na_cj_2log; + matrix box; + real rl2, rl_fep2 = 0; + float rbb2; + int ci_b, ci, ci_x, ci_y, ci_xy, cj; + ivec shp; + int shift; + real shx, shy, shz; + int conv_i, cell0_i; + const nbnxn_bb_t *bb_i = NULL; +#ifdef NBNXN_BBXXXX + const float *pbb_i = NULL; +#endif + const float *bbcz_i, *bbcz_j; + const int *flags_i; + real bx0, bx1, by0, by1, bz0, bz1; + real bz1_frac; + real d2cx, d2z, d2z_cx, d2z_cy, d2zx, d2zxy, d2xy; + int cxf, cxl, cyf, cyf_x, cyl; + int c0, c1, cs, cf, cl; + int ndistc; + int ncpcheck; + int gridi_flag_shift = 0, gridj_flag_shift = 0; + gmx_bitmask_t *gridj_flag = NULL; + int ncj_old_i, ncj_old_j; + + nbs_cycle_start(&work->cc[enbsCCsearch]); + + if (gridj->bSimple != nbl->bSimple) + { + gmx_incons("Grid incompatible with pair-list"); + } + + sync_work(nbl); + nbl->na_sc = gridj->na_sc; + nbl->na_ci = gridj->na_c; + nbl->na_cj = nbnxn_kernel_to_cluster_j_size(nb_kernel_type); + na_cj_2log = get_2log(nbl->na_cj); + + nbl->rlist = rlist; + + if (bFBufferFlag) + { + /* Determine conversion of clusters to flag blocks */ + gridi_flag_shift = 0; + while ((nbl->na_ci<na_cj<buffer_flags.flag; + } + + copy_mat(nbs->box, box); + + rl2 = nbl->rlist*nbl->rlist; + + if (nbs->bFEP && !nbl->bSimple) + { + /* Determine an atom-pair list cut-off distance for FEP atom pairs. + * We should not simply use rlist, since then we would not have + * the small, effective buffering of the NxN lists. + * The buffer is on overestimate, but the resulting cost for pairs + * beyond rlist is neglible compared to the FEP pairs within rlist. + */ + rl_fep2 = nbl->rlist + effective_buffer_1x1_vs_MxN(gridi, gridj); + + if (debug) + { + fprintf(debug, "nbl_fep atom-pair rlist %f\n", rl_fep2); + } + rl_fep2 = rl_fep2*rl_fep2; + } + + rbb2 = boundingbox_only_distance2(gridi, gridj, nbl->rlist, nbl->bSimple); + + if (debug) + { + fprintf(debug, "nbl bounding box only distance %f\n", std::sqrt(rbb2)); + } + + /* Set the shift range */ + for (int d = 0; d < DIM; d++) + { + /* Check if we need periodicity shifts. + * Without PBC or with domain decomposition we don't need them. + */ + if (d >= ePBC2npbcdim(nbs->ePBC) || nbs->dd_dim[d]) + { + shp[d] = 0; + } + else + { + if (d == XX && + box[XX][XX] - fabs(box[YY][XX]) - fabs(box[ZZ][XX]) < std::sqrt(rl2)) + { + shp[d] = 2; + } + else + { + shp[d] = 1; + } + } + } + + if (nbl->bSimple && !gridi->bSimple) + { + conv_i = gridi->na_sc/gridj->na_sc; + bb_i = gridi->bb_simple; + bbcz_i = gridi->bbcz_simple; + flags_i = gridi->flags_simple; + } + else + { + conv_i = 1; +#ifdef NBNXN_BBXXXX + if (gridi->bSimple) + { + bb_i = gridi->bb; + } + else + { + pbb_i = gridi->pbb; + } +#else + /* We use the normal bounding box format for both grid types */ + bb_i = gridi->bb; +#endif + bbcz_i = gridi->bbcz; + flags_i = gridi->flags; + } + cell0_i = gridi->cell0*conv_i; + + bbcz_j = gridj->bbcz; + + if (conv_i != 1) + { + /* Blocks of the conversion factor - 1 give a large repeat count + * combined with a small block size. This should result in good + * load balancing for both small and large domains. + */ + ci_block = conv_i - 1; + } + if (debug) + { + fprintf(debug, "nbl nc_i %d col.av. %.1f ci_block %d\n", + gridi->nc, gridi->nc/(double)(gridi->ncx*gridi->ncy), ci_block); + } + + ndistc = 0; + ncpcheck = 0; + + /* Initially ci_b and ci to 1 before where we want them to start, + * as they will both be incremented in next_ci. + */ + ci_b = -1; + ci = th*ci_block - 1; + ci_x = 0; + ci_y = 0; + while (next_ci(gridi, conv_i, nth, ci_block, &ci_x, &ci_y, &ci_b, &ci)) + { + if (nbl->bSimple && flags_i[ci] == 0) + { + continue; + } + + ncj_old_i = nbl->ncj; + + d2cx = 0; + if (gridj != gridi && shp[XX] == 0) + { + if (nbl->bSimple) + { + bx1 = bb_i[ci].upper[BB_X]; + } + else + { + bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx; + } + if (bx1 < gridj->c0[XX]) + { + d2cx = gmx::square(gridj->c0[XX] - bx1); + + if (d2cx >= rl2) + { + continue; + } + } + } + + ci_xy = ci_x*gridi->ncy + ci_y; + + /* Loop over shift vectors in three dimensions */ + for (int tz = -shp[ZZ]; tz <= shp[ZZ]; tz++) + { + shz = tz*box[ZZ][ZZ]; + + bz0 = bbcz_i[ci*NNBSBB_D ] + shz; + bz1 = bbcz_i[ci*NNBSBB_D+1] + shz; + + if (tz == 0) + { + d2z = 0; + } + else if (tz < 0) + { + d2z = gmx::square(bz1); + } + else + { + d2z = gmx::square(bz0 - box[ZZ][ZZ]); + } + + d2z_cx = d2z + d2cx; + + if (d2z_cx >= rl2) + { + continue; + } + + bz1_frac = bz1/(gridi->cxy_ind[ci_xy+1] - gridi->cxy_ind[ci_xy]); + if (bz1_frac < 0) + { + bz1_frac = 0; + } + /* The check with bz1_frac close to or larger than 1 comes later */ + + for (int ty = -shp[YY]; ty <= shp[YY]; ty++) + { + shy = ty*box[YY][YY] + tz*box[ZZ][YY]; + + if (nbl->bSimple) + { + by0 = bb_i[ci].lower[BB_Y] + shy; + by1 = bb_i[ci].upper[BB_Y] + shy; + } + else + { + by0 = gridi->c0[YY] + (ci_y )*gridi->sy + shy; + by1 = gridi->c0[YY] + (ci_y+1)*gridi->sy + shy; + } + + get_cell_range(by0, by1, + gridj->ncy, gridj->c0[YY], gridj->sy, gridj->inv_sy, + d2z_cx, rl2, + &cyf, &cyl); + + if (cyf > cyl) + { + continue; + } + + d2z_cy = d2z; + if (by1 < gridj->c0[YY]) + { + d2z_cy += gmx::square(gridj->c0[YY] - by1); + } + else if (by0 > gridj->c1[YY]) + { + d2z_cy += gmx::square(by0 - gridj->c1[YY]); + } + + for (int tx = -shp[XX]; tx <= shp[XX]; tx++) + { + shift = XYZ2IS(tx, ty, tz); + +#ifdef NBNXN_SHIFT_BACKWARD + if (gridi == gridj && shift > CENTRAL) + { + continue; + } +#endif + + shx = tx*box[XX][XX] + ty*box[YY][XX] + tz*box[ZZ][XX]; + + if (nbl->bSimple) + { + bx0 = bb_i[ci].lower[BB_X] + shx; + bx1 = bb_i[ci].upper[BB_X] + shx; + } + else + { + bx0 = gridi->c0[XX] + (ci_x )*gridi->sx + shx; + bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx + shx; + } + + get_cell_range(bx0, bx1, + gridj->ncx, gridj->c0[XX], gridj->sx, gridj->inv_sx, + d2z_cy, rl2, + &cxf, &cxl); + + if (cxf > cxl) + { + continue; + } + + if (nbl->bSimple) + { + new_ci_entry(nbl, cell0_i+ci, shift, flags_i[ci]); + } + else + { + new_sci_entry(nbl, cell0_i+ci, shift); + } + +#ifndef NBNXN_SHIFT_BACKWARD + if (cxf < ci_x) +#else + if (shift == CENTRAL && gridi == gridj && + cxf < ci_x) +#endif + { + /* Leave the pairs with i > j. + * x is the major index, so skip half of it. + */ + cxf = ci_x; + } + + if (nbl->bSimple) + { + set_icell_bb_simple(bb_i, ci, shx, shy, shz, + nbl->work->bb_ci); + } + else + { +#ifdef NBNXN_BBXXXX + set_icell_bbxxxx_supersub(pbb_i, ci, shx, shy, shz, + nbl->work->pbb_ci); +#else + set_icell_bb_supersub(bb_i, ci, shx, shy, shz, + nbl->work->bb_ci); +#endif + } + + nbs->icell_set_x(cell0_i+ci, shx, shy, shz, + nbat->xstride, nbat->x, + nbl->work); + + for (int cx = cxf; cx <= cxl; cx++) + { + d2zx = d2z; + if (gridj->c0[XX] + cx*gridj->sx > bx1) + { + d2zx += gmx::square(gridj->c0[XX] + cx*gridj->sx - bx1); + } + else if (gridj->c0[XX] + (cx+1)*gridj->sx < bx0) + { + d2zx += gmx::square(gridj->c0[XX] + (cx+1)*gridj->sx - bx0); + } + +#ifndef NBNXN_SHIFT_BACKWARD + if (gridi == gridj && + cx == 0 && cyf < ci_y) +#else + if (gridi == gridj && + cx == 0 && shift == CENTRAL && cyf < ci_y) +#endif + { + /* Leave the pairs with i > j. + * Skip half of y when i and j have the same x. + */ + cyf_x = ci_y; + } + else + { + cyf_x = cyf; + } + + for (int cy = cyf_x; cy <= cyl; cy++) + { + c0 = gridj->cxy_ind[cx*gridj->ncy+cy]; + c1 = gridj->cxy_ind[cx*gridj->ncy+cy+1]; +#ifdef NBNXN_SHIFT_BACKWARD + if (gridi == gridj && + shift == CENTRAL && c0 < ci) + { + c0 = ci; + } +#endif + + d2zxy = d2zx; + if (gridj->c0[YY] + cy*gridj->sy > by1) + { + d2zxy += gmx::square(gridj->c0[YY] + cy*gridj->sy - by1); + } + else if (gridj->c0[YY] + (cy+1)*gridj->sy < by0) + { + d2zxy += gmx::square(gridj->c0[YY] + (cy+1)*gridj->sy - by0); + } + if (c1 > c0 && d2zxy < rl2) + { + cs = c0 + static_cast(bz1_frac*(c1 - c0)); + if (cs >= c1) + { + cs = c1 - 1; + } + + d2xy = d2zxy - d2z; + + /* Find the lowest cell that can possibly + * be within range. + */ + cf = cs; + while (cf > c0 && + (bbcz_j[cf*NNBSBB_D+1] >= bz0 || + d2xy + gmx::square(bbcz_j[cf*NNBSBB_D+1] - bz0) < rl2)) + { + cf--; + } + + /* Find the highest cell that can possibly + * be within range. + */ + cl = cs; + while (cl < c1-1 && + (bbcz_j[cl*NNBSBB_D] <= bz1 || + d2xy + gmx::square(bbcz_j[cl*NNBSBB_D] - bz1) < rl2)) + { + cl++; + } + +#ifdef NBNXN_REFCODE + { + /* Simple reference code, for debugging, + * overrides the more complex code above. + */ + cf = c1; + cl = -1; + for (int k = c0; k < c1; k++) + { + if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 && + k < cf) + { + cf = k; + } + if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 && + k > cl) + { + cl = k; + } + } + } +#endif + + if (gridi == gridj) + { + /* We want each atom/cell pair only once, + * only use cj >= ci. + */ +#ifndef NBNXN_SHIFT_BACKWARD + cf = std::max(cf, ci); +#else + if (shift == CENTRAL) + { + cf = std::max(cf, ci); + } +#endif + } + + if (cf <= cl) + { + /* For f buffer flags with simple lists */ + ncj_old_j = nbl->ncj; + + switch (nb_kernel_type) + { + case nbnxnk4x4_PlainC: + check_cell_list_space_simple(nbl, cl-cf+1); + + make_cluster_list_simple(gridj, + nbl, ci, cf, cl, + (gridi == gridj && shift == CENTRAL), + nbat->x, + rl2, rbb2, + &ndistc); + break; +#ifdef GMX_NBNXN_SIMD_4XN + case nbnxnk4xN_SIMD_4xN: + check_cell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2); + make_cluster_list_simd_4xn(gridj, + nbl, ci, cf, cl, + (gridi == gridj && shift == CENTRAL), + nbat->x, + rl2, rbb2, + &ndistc); + break; +#endif +#ifdef GMX_NBNXN_SIMD_2XNN + case nbnxnk4xN_SIMD_2xNN: + check_cell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2); + make_cluster_list_simd_2xnn(gridj, + nbl, ci, cf, cl, + (gridi == gridj && shift == CENTRAL), + nbat->x, + rl2, rbb2, + &ndistc); + break; +#endif + case nbnxnk8x8x8_PlainC: + case nbnxnk8x8x8_GPU: + check_cell_list_space_supersub(nbl, cl-cf+1); + for (cj = cf; cj <= cl; cj++) + { + make_cluster_list_supersub(gridi, gridj, + nbl, ci, cj, + (gridi == gridj && shift == CENTRAL && ci == cj), + nbat->xstride, nbat->x, + rl2, rbb2, + &ndistc); + } + break; + } + ncpcheck += cl - cf + 1; + + if (bFBufferFlag && nbl->ncj > ncj_old_j) + { + int cbf = nbl->cj[ncj_old_j].cj >> gridj_flag_shift; + int cbl = nbl->cj[nbl->ncj-1].cj >> gridj_flag_shift; + for (int cb = cbf; cb <= cbl; cb++) + { + bitmask_init_bit(&gridj_flag[cb], th); + } + } + } + } + } + } + + /* Set the exclusions for this ci list */ + if (nbl->bSimple) + { + set_ci_top_excls(nbs, + nbl, + shift == CENTRAL && gridi == gridj, + gridj->na_c_2log, + na_cj_2log, + &(nbl->ci[nbl->nci]), + excl); + + if (nbs->bFEP) + { + make_fep_list(nbs, nbat, nbl, + shift == CENTRAL && gridi == gridj, + &(nbl->ci[nbl->nci]), + gridi, gridj, nbl_fep); + } + } + else + { + set_sci_top_excls(nbs, + nbl, + shift == CENTRAL && gridi == gridj, + gridj->na_c_2log, + &(nbl->sci[nbl->nsci]), + excl); + + if (nbs->bFEP) + { + make_fep_list_supersub(nbs, nbat, nbl, + shift == CENTRAL && gridi == gridj, + &(nbl->sci[nbl->nsci]), + shx, shy, shz, + rl_fep2, + gridi, gridj, nbl_fep); + } + } + + /* Close this ci list */ + if (nbl->bSimple) + { + close_ci_entry_simple(nbl); + } + else + { + close_ci_entry_supersub(nbl, + nsubpair_max, + progBal, nsubpair_tot_est, + th, nth); + } + } + } + } + + if (bFBufferFlag && nbl->ncj > ncj_old_i) + { + bitmask_init_bit(&(work->buffer_flags.flag[(gridi->cell0+ci)>>gridi_flag_shift]), th); + } + } + + work->ndistc = ndistc; + + nbs_cycle_stop(&work->cc[enbsCCsearch]); + + if (debug) + { + fprintf(debug, "number of distance checks %d\n", ndistc); + fprintf(debug, "ncpcheck %s %d\n", gridi == gridj ? "local" : "non-local", + ncpcheck); + + if (nbl->bSimple) + { + print_nblist_statistics_simple(debug, nbl, nbs, rlist); + } + else + { + print_nblist_statistics_supersub(debug, nbl, nbs, rlist); + } + + if (nbs->bFEP) + { + fprintf(debug, "nbl FEP list pairs: %d\n", nbl_fep->nrj); + } + } +} + +static void reduce_buffer_flags(const nbnxn_search_t nbs, + int nsrc, + const nbnxn_buffer_flags_t *dest) +{ + for (int s = 0; s < nsrc; s++) + { + gmx_bitmask_t * flag = nbs->work[s].buffer_flags.flag; + + for (int b = 0; b < dest->nflag; b++) + { + bitmask_union(&(dest->flag[b]), flag[b]); + } + } +} + +static void print_reduction_cost(const nbnxn_buffer_flags_t *flags, int nout) +{ + int nelem, nkeep, ncopy, nred, out; + gmx_bitmask_t mask_0; + + nelem = 0; + nkeep = 0; + ncopy = 0; + nred = 0; + bitmask_init_bit(&mask_0, 0); + for (int b = 0; b < flags->nflag; b++) + { + if (bitmask_is_equal(flags->flag[b], mask_0)) + { + /* Only flag 0 is set, no copy of reduction required */ + nelem++; + nkeep++; + } + else if (!bitmask_is_zero(flags->flag[b])) + { + int c = 0; + for (out = 0; out < nout; out++) + { + if (bitmask_is_set(flags->flag[b], out)) + { + c++; + } + } + nelem += c; + if (c == 1) + { + ncopy++; + } + else + { + nred += c; + } + } + } + + fprintf(debug, "nbnxn reduction: #flag %d #list %d elem %4.2f, keep %4.2f copy %4.2f red %4.2f\n", + flags->nflag, nout, + nelem/(double)(flags->nflag), + nkeep/(double)(flags->nflag), + ncopy/(double)(flags->nflag), + nred/(double)(flags->nflag)); +} + +/* Perform a count (linear) sort to sort the smaller lists to the end. + * This avoids load imbalance on the GPU, as large lists will be + * scheduled and executed first and the smaller lists later. + * Load balancing between multi-processors only happens at the end + * and there smaller lists lead to more effective load balancing. + * The sorting is done on the cj4 count, not on the actual pair counts. + * Not only does this make the sort faster, but it also results in + * better load balancing than using a list sorted on exact load. + * This function swaps the pointer in the pair list to avoid a copy operation. + */ +static void sort_sci(nbnxn_pairlist_t *nbl) +{ + nbnxn_list_work_t *work; + int m, s0, s1; + nbnxn_sci_t *sci_sort; + + if (nbl->ncj4 <= nbl->nsci) + { + /* nsci = 0 or all sci have size 1, sorting won't change the order */ + return; + } + + work = nbl->work; + + /* We will distinguish differences up to double the average */ + m = (2*nbl->ncj4)/nbl->nsci; + + if (m + 1 > work->sort_nalloc) + { + work->sort_nalloc = over_alloc_large(m + 1); + srenew(work->sort, work->sort_nalloc); + } + + if (work->sci_sort_nalloc != nbl->sci_nalloc) + { + work->sci_sort_nalloc = nbl->sci_nalloc; + nbnxn_realloc_void((void **)&work->sci_sort, + 0, + work->sci_sort_nalloc*sizeof(*work->sci_sort), + nbl->alloc, nbl->free); + } + + /* Count the entries of each size */ + for (int i = 0; i <= m; i++) + { + work->sort[i] = 0; + } + for (int s = 0; s < nbl->nsci; s++) + { + int i = std::min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start); + work->sort[i]++; + } + /* Calculate the offset for each count */ + s0 = work->sort[m]; + work->sort[m] = 0; + for (int i = m - 1; i >= 0; i--) + { + s1 = work->sort[i]; + work->sort[i] = work->sort[i + 1] + s0; + s0 = s1; + } + + /* Sort entries directly into place */ + sci_sort = work->sci_sort; + for (int s = 0; s < nbl->nsci; s++) + { + int i = std::min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start); + sci_sort[work->sort[i]++] = nbl->sci[s]; + } + + /* Swap the sci pointers so we use the new, sorted list */ + work->sci_sort = nbl->sci; + nbl->sci = sci_sort; +} + +/* Make a local or non-local pair-list, depending on iloc */ +void nbnxn_make_pairlist(const nbnxn_search_t nbs, + nbnxn_atomdata_t *nbat, + const t_blocka *excl, + real rlist, + int min_ci_balanced, + nbnxn_pairlist_set_t *nbl_list, + int iloc, + int nb_kernel_type, + t_nrnb *nrnb) +{ + nbnxn_grid_t *gridi, *gridj; + gmx_bool bGPUCPU; + int nzi, zj0, zj1; + int nsubpair_target, nsubpair_tot_est; + int nnbl; + nbnxn_pairlist_t **nbl; + int ci_block; + gmx_bool CombineNBLists; + gmx_bool progBal; + int np_tot, np_noq, np_hlj, nap; + + /* Check if we are running hybrid GPU + CPU nbnxn mode */ + bGPUCPU = (!nbs->grid[0].bSimple && nbl_list->bSimple); + + nnbl = nbl_list->nnbl; + nbl = nbl_list->nbl; + CombineNBLists = nbl_list->bCombined; + + if (debug) + { + fprintf(debug, "ns making %d nblists\n", nnbl); + } + + nbat->bUseBufferFlags = (nbat->nout > 1); + /* We should re-init the flags before making the first list */ + if (nbat->bUseBufferFlags && (LOCAL_I(iloc) || bGPUCPU)) + { + init_buffer_flags(&nbat->buffer_flags, nbat->natoms); + } + + if (nbl_list->bSimple) + { +#if GMX_SIMD + switch (nb_kernel_type) + { +#ifdef GMX_NBNXN_SIMD_4XN + case nbnxnk4xN_SIMD_4xN: + nbs->icell_set_x = icell_set_x_simd_4xn; + break; +#endif +#ifdef GMX_NBNXN_SIMD_2XNN + case nbnxnk4xN_SIMD_2xNN: + nbs->icell_set_x = icell_set_x_simd_2xnn; + break; +#endif + default: + nbs->icell_set_x = icell_set_x_simple; + break; + } +#else // GMX_SIMD + /* MSVC 2013 complains about switch statements without case */ + nbs->icell_set_x = icell_set_x_simple; +#endif // GMX_SIMD + } + else + { + nbs->icell_set_x = icell_set_x_supersub; + } + + if (LOCAL_I(iloc)) + { + /* Only zone (grid) 0 vs 0 */ + nzi = 1; + zj0 = 0; + zj1 = 1; + } + else + { + nzi = nbs->zones->nizone; + } + + if (!nbl_list->bSimple && min_ci_balanced > 0) + { + get_nsubpair_target(nbs, iloc, rlist, min_ci_balanced, + &nsubpair_target, &nsubpair_tot_est); + } + else + { + nsubpair_target = 0; + nsubpair_tot_est = 0; + } + + /* Clear all pair-lists */ + for (int th = 0; th < nnbl; th++) + { + clear_pairlist(nbl[th]); + + if (nbs->bFEP) + { + clear_pairlist_fep(nbl_list->nbl_fep[th]); + } + } + + for (int zi = 0; zi < nzi; zi++) + { + gridi = &nbs->grid[zi]; + + if (NONLOCAL_I(iloc)) + { + zj0 = nbs->zones->izone[zi].j0; + zj1 = nbs->zones->izone[zi].j1; + if (zi == 0) + { + zj0++; + } + } + for (int zj = zj0; zj < zj1; zj++) + { + gridj = &nbs->grid[zj]; + + if (debug) + { + fprintf(debug, "ns search grid %d vs %d\n", zi, zj); + } + + nbs_cycle_start(&nbs->cc[enbsCCsearch]); + + if (nbl[0]->bSimple && !gridi->bSimple) + { + /* Hybrid list, determine blocking later */ + ci_block = 0; + } + else + { + ci_block = get_ci_block_size(gridi, nbs->DomDec, nnbl); + } + + /* With GPU: generate progressively smaller lists for + * load balancing for local only or non-local with 2 zones. + */ + progBal = (LOCAL_I(iloc) || nbs->zones->n <= 2); + +#pragma omp parallel for num_threads(nnbl) schedule(static) + for (int th = 0; th < nnbl; th++) + { + try + { + /* Re-init the thread-local work flag data before making + * the first list (not an elegant conditional). + */ + if (nbat->bUseBufferFlags && ((zi == 0 && zj == 0) || + (bGPUCPU && zi == 0 && zj == 1))) + { + init_buffer_flags(&nbs->work[th].buffer_flags, nbat->natoms); + } + + if (CombineNBLists && th > 0) + { + clear_pairlist(nbl[th]); + } + + /* Divide the i super cell equally over the nblists */ + nbnxn_make_pairlist_part(nbs, gridi, gridj, + &nbs->work[th], nbat, excl, + rlist, + nb_kernel_type, + ci_block, + nbat->bUseBufferFlags, + nsubpair_target, + progBal, nsubpair_tot_est, + th, nnbl, + nbl[th], + nbl_list->nbl_fep[th]); + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + } + nbs_cycle_stop(&nbs->cc[enbsCCsearch]); + + np_tot = 0; + np_noq = 0; + np_hlj = 0; + for (int th = 0; th < nnbl; th++) + { + inc_nrnb(nrnb, eNR_NBNXN_DIST2, nbs->work[th].ndistc); + + if (nbl_list->bSimple) + { + np_tot += nbl[th]->ncj; + np_noq += nbl[th]->work->ncj_noq; + np_hlj += nbl[th]->work->ncj_hlj; + } + else + { + /* This count ignores potential subsequent pair pruning */ + np_tot += nbl[th]->nci_tot; + } + } + nap = nbl[0]->na_ci*nbl[0]->na_cj; + nbl_list->natpair_ljq = (np_tot - np_noq)*nap - np_hlj*nap/2; + nbl_list->natpair_lj = np_noq*nap; + nbl_list->natpair_q = np_hlj*nap/2; + + if (CombineNBLists && nnbl > 1) + { + nbs_cycle_start(&nbs->cc[enbsCCcombine]); + + combine_nblists(nnbl-1, nbl+1, nbl[0]); + + nbs_cycle_stop(&nbs->cc[enbsCCcombine]); + } + } + } + + if (!nbl_list->bSimple) + { + /* Sort the entries on size, large ones first */ + if (CombineNBLists || nnbl == 1) + { + sort_sci(nbl[0]); + } + else + { +#pragma omp parallel for num_threads(nnbl) schedule(static) + for (int th = 0; th < nnbl; th++) + { + try + { + sort_sci(nbl[th]); + } + GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR; + } + } + } + + if (nbat->bUseBufferFlags) + { + reduce_buffer_flags(nbs, nnbl, &nbat->buffer_flags); + } + + if (nbs->bFEP) + { + /* Balance the free-energy lists over all the threads */ + balance_fep_lists(nbs, nbl_list); + } + + /* Special performance logging stuff (env.var. GMX_NBNXN_CYCLE) */ + if (LOCAL_I(iloc)) + { + nbs->search_count++; + } + if (nbs->print_cycles && + (!nbs->DomDec || (nbs->DomDec && !LOCAL_I(iloc))) && + nbs->search_count % 100 == 0) + { + nbs_cycle_print(stderr, nbs); + } + + if (debug && (CombineNBLists && nnbl > 1)) + { + if (nbl[0]->bSimple) + { + print_nblist_statistics_simple(debug, nbl[0], nbs, rlist); + } + else + { + print_nblist_statistics_supersub(debug, nbl[0], nbs, rlist); + } + } + + if (debug) + { + if (gmx_debug_at) + { + if (nbl[0]->bSimple) + { + print_nblist_ci_cj(debug, nbl[0]); + } + else + { + print_nblist_sci_cj(debug, nbl[0]); + } + } + + if (nbat->bUseBufferFlags) + { + print_reduction_cost(&nbat->buffer_flags, nnbl); + } + } +}