From: Paul Bauer Date: Wed, 7 Oct 2020 06:55:19 +0000 (+0200) Subject: Merge branch 'origin/release-2020' into merge-release-2020-into-master X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=694edca8e684c5efd910e19e1a685827a258b2ba;p=alexxy%2Fgromacs.git Merge branch 'origin/release-2020' into merge-release-2020-into-master Resolved Conflicts: admin/gitlab-ci/archive.gitlab-ci.yml admin/gitlab-ci/global.gitlab-ci.yml admin/gitlab-ci/gromacs.gitlab-ci.yml admin/gitlab-ci/lint.gitlab-ci.yml admin/gitlab-ci/python-gmxapi.gitlab-ci.yml admin/gitlab-ci/rules.gitlab-ci.yml admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml cmake/gmxVersionInfo.cmake src/gromacs/fileio/checkpoint.cpp src/gromacs/hardware/printhardware.cpp src/gromacs/mdlib/md_support.cpp src/gromacs/mdlib/trajectory_writing.cpp src/gromacs/mdrun/runner.cpp src/gromacs/modularsimulator/domdechelper.cpp src/gromacs/modularsimulator/domdechelper.h src/gromacs/modularsimulator/freeenergyperturbationelement.cpp src/gromacs/modularsimulator/freeenergyperturbationelement.h src/gromacs/modularsimulator/modularsimulator.cpp src/gromacs/tools/trjcat.cpp src/gromacs/topology/topology.h src/gromacs/utility/fatalerror.cpp src/gromacs/utility/futil.cpp src/gromacs/utility/init.cpp src/programs/mdrun/tests/CMakeLists.txt tests/CMakeLists.txt Change-Id: Icd5d9c78ff2cfb0598c3cb55b057487ca098a1f0 --- 694edca8e684c5efd910e19e1a685827a258b2ba diff --cc admin/gitlab-ci/global.gitlab-ci.yml index d6dbf6bf09,302a58e05e..49056b4959 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@@ -11,10 -11,11 +11,11 @@@ variables: KUBERNETES_CPU_LIMIT: 8 KUBERNETES_CPU_REQUEST: 4 - KUBERNETES_MEMORY_REQUEST: 8Gi + KUBERNETES_MEMORY_REQUEST: 4Gi + KUBERNETES_MEMORY_LIMIT: 8Gi KUBERNETES_EXTENDED_RESOURCE_NAME: "" KUBERNETES_EXTENDED_RESOURCE_LIMIT: 0 - CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-release-2020" + CACHE_FALLBACK_KEY: "$CI_JOB_NAME-$CI_JOB_STAGE-master" BUILD_DIR: build INSTALL_DIR: install CMAKE_GMXAPI_OPTIONS: "" diff --cc admin/gitlab-ci/rules.gitlab-ci.yml index 201e329599,86bf421545..7bf1648b47 --- a/admin/gitlab-ci/rules.gitlab-ci.yml +++ b/admin/gitlab-ci/rules.gitlab-ci.yml @@@ -73,13 -73,13 +73,13 @@@ .rules-element:if-post-merge-acceptance-or-mr-then-always: &if-post-merge-acceptance-or-mr-then-always if: '$CI_PIPELINE_SOURCE == "merge_request_event" || ($CI_PIPELINE_SOURCE == "push" && - ($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-*"))' - $CI_COMMIT_REF_NAME == "release-2020")' ++ $CI_COMMIT_REF_NAME == "master")' when: always # Include job only for post submit push .rules-element:if-post-merge-acceptance-then-always: &if-post-merge-acceptance-then-always if: '$CI_PIPELINE_SOURCE == "push" && - ($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "release-*")' - $CI_COMMIT_REF_NAME == "release-2020"' ++ $CI_COMMIT_REF_NAME == "master"' when: always # When composing a rule set, note that the first matching rule is applied. diff --cc cmake/gmxTestImageMagick.cmake index 2912b85f3b,bc06d5af8f..b008a79170 --- a/cmake/gmxTestImageMagick.cmake +++ b/cmake/gmxTestImageMagick.cmake @@@ -1,7 -1,7 +1,7 @@@ # # This file is part of the GROMACS molecular simulation package. # --# Copyright (c) 2018,2019, by the GROMACS development team, led by ++# Copyright (c) 2018,2019,2020, by the GROMACS development team, led by # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, # and including many others, as listed in the AUTHORS file in the # top-level source directory and at http://www.gromacs.org. diff --cc docs/doxygen/suppressions.txt index c5d844eefc,533e028090..6a976c4971 --- a/docs/doxygen/suppressions.txt +++ b/docs/doxygen/suppressions.txt @@@ -47,10 -44,9 +47,9 @@@ src/gromacs/tables/cubicsplinetable.h: src/gromacs/tables/quadraticsplinetable.h: warning: includes "simd.h" unnecessarily # These are specific to Folding@Home, and easiest to suppress here - *: warning: includes non-local file as "corewrap.h" src/gmxpre.h: warning: includes non-local file as "swindirect.h" -# New external API (see https://redmine.gromacs.org/issues/2586) has some unresolved +# New external API (see https://gitlab.com/gromacs/gromacs/-/issues/2586) has some unresolved # conflicts with previous definitions of public API, installed API, and other things # described or implemented in check-source.py, gmxtree.py, gmxtree.rst, and others # TODO: resolve definitions, update testing heuristics, and activate policy checks diff --cc src/gromacs/fileio/checkpoint.cpp index 39d5b2984a,9c6cfe4213..abd957f09a --- a/src/gromacs/fileio/checkpoint.cpp +++ b/src/gromacs/fileio/checkpoint.cpp @@@ -2342,13 -2393,94 +2338,29 @@@ void write_checkpoint_data(t_fileio gmx::serializeKeyValueTree(tree, &serializer); } - do_cpt_footer(gmx_fio_getxdr(fp), headerContents.file_version); - - /* we really, REALLY, want to make sure to physically write the checkpoint, - and all the files it depends on, out to disk. Because we've - opened the checkpoint with gmx_fio_open(), it's in our list - of open files. */ - ret = gmx_fio_all_output_fsync(); - - if (ret) + // Checkpointing modular simulator { - char buf[STRLEN]; - sprintf(buf, "Cannot fsync '%s'; maybe you are out of disk space?", gmx_fio_getname(ret)); - - if (getenv(GMX_IGNORE_FSYNC_FAILURE_ENV) == nullptr) - { - gmx_file(buf); - } - else - { - gmx_warning("%s", buf); - } - } - - if (gmx_fio_close(fp) != 0) - { - gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?"); - } - - /* we don't move the checkpoint if the user specified they didn't want it, - or if the fsyncs failed */ -#if !GMX_NO_RENAME - if (!bNumberAndKeep && !ret) - { - if (gmx_fexist(fn)) - { - /* Rename the previous checkpoint file */ - mpiBarrierBeforeRename(applyMpiBarrierBeforeRename, mpiBarrierCommunicator); - - std::strcpy(buf, fn); - buf[std::strlen(fn) - std::strlen(ftp2ext(fn2ftp(fn))) - 1] = '\0'; - std::strcat(buf, "_prev"); - std::strcat(buf, fn + std::strlen(fn) - std::strlen(ftp2ext(fn2ftp(fn))) - 1); - if (!GMX_FAHCORE) - { - /* we copy here so that if something goes wrong between now and - * the rename below, there's always a state.cpt. - * If renames are atomic (such as in POSIX systems), - * this copying should be unneccesary. - */ - gmx_file_copy(fn, buf, FALSE); - /* We don't really care if this fails: - * there's already a new checkpoint. - */ - } - else - { - gmx_file_rename(fn, buf); - } - } - - /* Rename the checkpoint file from the temporary to the final name */ - mpiBarrierBeforeRename(applyMpiBarrierBeforeRename, mpiBarrierCommunicator); - - if (gmx_file_rename(fntemp, fn) != 0) - { - gmx_file("Cannot rename checkpoint file; maybe you are out of disk space?"); - } + gmx::FileIOXdrSerializer serializer(fp); + modularSimulatorCheckpointData->serialize(&serializer); } -#endif /* GMX_NO_RENAME */ - - sfree(fntemp); + do_cpt_footer(gmx_fio_getxdr(fp), headerContents.file_version); + #if GMX_FAHCORE + /* Always FAH checkpoint immediately after a Gromacs checkpoint. + * + * Note that it is critical that we save a FAH checkpoint directly + * after writing a Gromacs checkpoint. If the program dies, either + * by the machine powering off suddenly or the process being, + * killed, FAH can recover files that have only appended data by + * truncating them to the last recorded length. The Gromacs + * checkpoint does not just append data, it is fully rewritten each + * time so a crash between moving the new Gromacs checkpoint file in + * to place and writing a FAH checkpoint is not recoverable. Thus + * the time between these operations must be kept as short a + * possible. + */ + fcCheckpoint(); + #endif } static void check_int(FILE* fplog, const char* type, int p, int f, gmx_bool* mm) diff --cc src/gromacs/hardware/printhardware.cpp index 616b7b9e09,43f7023409..0720a12bbe --- a/src/gromacs/hardware/printhardware.cpp +++ b/src/gromacs/hardware/printhardware.cpp @@@ -60,8 -60,10 +60,10 @@@ #include "gromacs/utility/stringutil.h" #include "gromacs/utility/sysinfo.h" + #include "architecture.h" + //! Constant used to help minimize preprocessed code -static const bool bGPUBinary = GMX_GPU != GMX_GPU_NONE; +static constexpr bool bGPUBinary = (GMX_GPU != 0); /*! \internal \brief * Returns the GPU information text, one GPU per line. diff --cc src/gromacs/mdlib/trajectory_writing.cpp index d9ef7191ed,3d99d2353d..f0f9f25fda --- a/src/gromacs/mdlib/trajectory_writing.cpp +++ b/src/gromacs/mdlib/trajectory_writing.cpp @@@ -1,8 -1,7 +1,8 @@@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2013,2014,2015,2016,2017 by the GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016,2017, The GROMACS development team. + * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. diff --cc src/gromacs/mdrun/runner.cpp index dfc399541f,3c9def3eaa..90275f50a3 --- a/src/gromacs/mdrun/runner.cpp +++ b/src/gromacs/mdrun/runner.cpp @@@ -205,14 -190,13 +201,14 @@@ static DevelopmentFeatureFlags manageDe // getenv results are ignored when clearly they are used. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - devFlags.enableGpuBufferOps = (getenv("GMX_USE_GPU_BUFFER_OPS") != nullptr) - && (GMX_GPU == GMX_GPU_CUDA) && useGpuForNonbonded; + + devFlags.enableGpuBufferOps = + GMX_GPU_CUDA && useGpuForNonbonded && (getenv("GMX_USE_GPU_BUFFER_OPS") != nullptr); + devFlags.enableGpuHaloExchange = GMX_GPU_CUDA && GMX_THREAD_MPI && getenv("GMX_GPU_DD_COMMS") != nullptr; - devFlags.forceGpuUpdateDefault = (getenv("GMX_FORCE_UPDATE_DEFAULT_GPU") != nullptr); + devFlags.forceGpuUpdateDefault = (getenv("GMX_FORCE_UPDATE_DEFAULT_GPU") != nullptr) || GMX_FAHCORE; - devFlags.enableGpuHaloExchange = - (getenv("GMX_GPU_DD_COMMS") != nullptr && GMX_THREAD_MPI && (GMX_GPU == GMX_GPU_CUDA)); devFlags.enableGpuPmePPComm = - (getenv("GMX_GPU_PME_PP_COMMS") != nullptr && GMX_THREAD_MPI && (GMX_GPU == GMX_GPU_CUDA)); + GMX_GPU_CUDA && GMX_THREAD_MPI && getenv("GMX_GPU_PME_PP_COMMS") != nullptr; + #pragma GCC diagnostic pop if (devFlags.enableGpuBufferOps) @@@ -1060,26 -1007,32 +1049,41 @@@ int Mdrunner::mdrunner( * So the PME-only nodes (if present) will also initialize * the distance restraints. */ - snew(fcd, 1); /* This needs to be called before read_checkpoint to extend the state */ - init_disres(fplog, &mtop, inputrec, cr, ms, fcd, globalState.get(), replExParams.exchangeInterval > 0); + t_disresdata* disresdata; + snew(disresdata, 1); + init_disres(fplog, &mtop, inputrec.get(), DisResRunMode::MDRun, + MASTER(cr) ? DDRole::Master : DDRole::Agent, + PAR(cr) ? NumRanks::Multiple : NumRanks::Single, cr->mpi_comm_mysim, ms, disresdata, + globalState.get(), replExParams.exchangeInterval > 0); - init_orires(fplog, &mtop, inputrec, cr, ms, globalState.get(), &(fcd->orires)); + t_oriresdata* oriresdata; + snew(oriresdata, 1); + init_orires(fplog, &mtop, inputrec.get(), cr, ms, globalState.get(), oriresdata); - auto deform = prepareBoxDeformation(globalState->box, cr, *inputrec); + auto deform = prepareBoxDeformation( + globalState != nullptr ? globalState->box : box, MASTER(cr) ? DDRole::Master : DDRole::Agent, + PAR(cr) ? NumRanks::Multiple : NumRanks::Single, cr->mpi_comm_mygroup, *inputrec); + #if GMX_FAHCORE + /* We have to remember the generation's first step before reading checkpoint. + This way, we can report to the F@H core both the generation's first step + and the restored first step, thus making it able to distinguish between + an interruption/resume and start of the n-th generation simulation. + Having this information, the F@H core can correctly calculate and report + the progress. + */ + int gen_first_step = 0; + if (MASTER(cr)) + { + gen_first_step = inputrec->init_step; + } + #endif + ObservablesHistory observablesHistory = {}; + auto modularSimulatorCheckpointData = std::make_unique(); if (startingBehavior != StartingBehavior::NewSimulation) { /* Check if checkpoint file exists before doing continuation. diff --cc src/gromacs/tools/trjcat.cpp index 5b0b20b683,0d5e1f45cb..b2377eafd6 --- a/src/gromacs/tools/trjcat.cpp +++ b/src/gromacs/tools/trjcat.cpp @@@ -3,8 -3,7 +3,8 @@@ * * Copyright (c) 1991-2000, University of Groningen, The Netherlands. * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017 by the GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016,2017, The GROMACS development team. + * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. diff --cc src/gromacs/topology/topology.h index 6ac65d0b39,1d8f9abc21..cda4ec7e94 --- a/src/gromacs/topology/topology.h +++ b/src/gromacs/topology/topology.h @@@ -3,8 -3,7 +3,8 @@@ * * Copyright (c) 1991-2000, University of Groningen, The Netherlands. * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2011,2014,2015,2016,2018 by the GROMACS development team. - * Copyright (c) 2011,2014,2015,2016,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2011,2014,2015,2016,2018, The GROMACS development team. + * Copyright (c) 2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. diff --cc src/gromacs/utility/fatalerror.cpp index 04127783fd,e27f64b692..f29b400a80 --- a/src/gromacs/utility/fatalerror.cpp +++ b/src/gromacs/utility/fatalerror.cpp @@@ -3,8 -3,7 +3,8 @@@ * * Copyright (c) 1991-2000, University of Groningen, The Netherlands. * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017 by the GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016,2017, The GROMACS development team. + * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. diff --cc src/gromacs/utility/futil.cpp index b09c5403b8,c16605e90a..6f631efc8b --- a/src/gromacs/utility/futil.cpp +++ b/src/gromacs/utility/futil.cpp @@@ -3,8 -3,7 +3,8 @@@ * * Copyright (c) 1991-2000, University of Groningen, The Netherlands. * Copyright (c) 2001-2004, The GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017 by the GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016,2017, The GROMACS development team. + * Copyright (c) 2018,2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. @@@ -164,13 -163,9 +164,9 @@@ static void push_ps(FILE* fp # ifdef gmx_ffclose # undef gmx_ffclose # endif - # if (!HAVE_PIPES && !defined(__native_client__)) + #endif + #if (!HAVE_PIPES && !defined(__native_client__)) -static FILE* popen(const char* nm, const char* mode) +static FILE* popen(const char* /* nm */, const char* /* mode */) { gmx_impl("Sorry no pipes..."); @@@ -692,14 -678,13 +679,14 @@@ int gmx_fsync(FILE* fp int fn; /* get the file number */ - # if HAVE_FILENO + #if HAVE_FILENO fn = fileno(fp); - # elif HAVE__FILENO + #elif HAVE__FILENO fn = _fileno(fp); - # else + #else + GMX_UNUSED_VALUE(fp); fn = -1; - # endif + #endif /* do the actual fsync */ if (fn >= 0) diff --cc src/gromacs/utility/init.cpp index f23f8cae50,b9ee238517..1f6e07414b --- a/src/gromacs/utility/init.cpp +++ b/src/gromacs/utility/init.cpp @@@ -1,8 -1,7 +1,8 @@@ /* * This file is part of the GROMACS molecular simulation package. * - * Copyright (c) 2013,2014,2015,2016,2018 by the GROMACS development team. - * Copyright (c) 2013,2014,2015,2016,2018,2019,2020, by the GROMACS development team, led by ++ * Copyright (c) 2013,2014,2015,2016,2018, The GROMACS development team. + * Copyright (c) 2019,2020, by the GROMACS development team, led by * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, * and including many others, as listed in the AUTHORS file in the * top-level source directory and at http://www.gromacs.org. diff --cc src/programs/mdrun/tests/CMakeLists.txt index dee12e276b,9ba03c04fc..b051068c06 --- a/src/programs/mdrun/tests/CMakeLists.txt +++ b/src/programs/mdrun/tests/CMakeLists.txt @@@ -95,6 -92,6 +95,9 @@@ gmx_add_gtest_executable(${exename target_link_libraries(${exename} PRIVATE mdrun_test_infrastructure) gmx_register_gtest_test(${testname} ${exename} OPENMP_THREADS 2 INTEGRATION_TEST IGNORE_LEAKS) ++# To avoid running into test timeouts, some end-to-end tests of mdrun ++# functionality are split off. This can be rearranged in future as we ++# see fit. set(testname "MdrunTests") set(exename "mdrun-test")